{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9718ed38",
   "metadata": {},
   "source": [
    "### 环境初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "42cfdf6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 在jupyter notebook里env.render看不到窗口\n",
    "# 写一个helper类，用matplotlib刷新显示图像\n",
    "# 初始化传入env，调用helper的render即可\n",
    "from IPython import display # 导入display模块，用于在Jupyter Notebook中显示图像\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt # 导入matplotlib库，用于绘制图像\n",
    "%matplotlib inline\n",
    "\n",
    "class GymHelper:\n",
    "    def __init__(self, env, figsize = (3, 3)):\n",
    "        self.env = env # 初始化Gym环境\n",
    "        self.figsize = figsize # 初始化绘图窗口大小\n",
    "        \n",
    "        plt.figure(figsize = figsize) # 创建绘图窗口\n",
    "        plt.title(self.env.spec.id) # 标题设为环境名\n",
    "        self.img = plt.imshow(env.render()) # 在绘图窗口中显示初始图像\n",
    "    \n",
    "    def render(self, title = None):\n",
    "        image_data = self.env.render() # 获取当前环境图像渲染数据\n",
    "        \n",
    "        self.img.set_data(image_data) # 更新绘图窗口中的图像数据\n",
    "        display.display(plt.gcf()) # 刷新显示\n",
    "        display.clear_output(wait = True) # 有新图片时再清除绘图窗口原有图像\n",
    "        if title: # 如果有标题，就显示标题\n",
    "            plt.title(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e40560ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAMsAAADSCAYAAADkIjRgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8+yak3AAAACXBIWXMAAAsTAAALEwEAmpwYAAARBklEQVR4nO3dfZBddX3H8fdn7+5mn0KCIYZI0AhGMxSEhAwPxnEUSxtTq84UHRgG0TKDrdrR+hjajg+dtoOdjo9jS2l9QMcKVK0y+AAI+ICFQHgID6HIQhPlKSFAstndZJ/ut3/cs+nNunv3t7vn3rv37uc1c2fPPefuPd/N5rPn/M4593wVEZjZ9FrqXYBZo3BYzBI5LGaJHBazRA6LWSKHxSyRw1JnkvolnZDXayW9XtIT+VQHkkLSK/J6v0bWWu8CFhJJO4EVwFjZ7FdGxFMp3x8RPdWoy9I4LLX3xxHx03oXYTPn3bA6K9/NkfR1SV+W9ENJByRtlXTiFK/dLGlH9ronJX1kwvt+WNIeSU9LenfZ/EWS/knSbyTtlnSFpM6y5R/NvucpSX9a/X+BxuGwzD/nA58GjgZ6gb+f4nVfAd4TEYuBk4FbypYdCywBjgMuAb4s6ehs2eXAK4HTgFdkr/kEgKRNwEeAc4E1wO/n9UM1A4el9r4vaV/2+P4ky/8rIu6MiFHgW5T+U09mBDhJ0lER8UJE3DNh2d9GxEhE/AjoB14lScClwF9GxPMRcQD4B0oBBXgH8LWIeDAiBoBPzfFnbSoOS+29LSKWZo+3TbL8mbLpQWCqQf2fAJuBXZJ+LunssmXPZWGb+D7LgS7g7vHAAj/J5gO8BPht2fftSvyZFgQP8BtURNwFvFVSG/B+4Frg+Gm+bS9wEPi9iHhykuVPT3iPl+ZRa7PwlqUBSWqXdKGkJRExAvQBxem+LyKKwL8Bn5P04uy9jpP0h9lLrgXeJekkSV3AJ6v0IzQkh6VxXQTslNQH/BlwYeL3fZzSgYM7su/9KfAqgIj4MfB5SgcLejnyoMGCJ3/4yyyNtyxmiaoSFkmbJD0iqVfSlmqsw6zWct8Nk1QAfk3pxNYTwF3ABRGxI9cVmdVYNbYsZwC9EfF4RAwDVwNvrcJ6zGqqGmE5jiNPbD2RzTNraHU7KSnpUkqXXtDd3X362rVr61WK2WE7d+5k7969mmxZNcLyJEeeBV6VzTtCRFwJXAmwYcOG2LZtWxVKMZuZDRs2TLmsGrthdwFrJL1cUjuli/Suq8J6zGoq9y1LRIxKej9wA1AAvhoRD+W9HrNaq8qYJbss/EfVeG+zevEZfLNEDotZIofFLJHDYpbIYTFL5LCYJXJYzBI5LGaJHBazRA6LWSKHxSyRw2KWyGExS+SwmCVyWMwSOSxmiRwWs0QOi1kih8UskcNilshhMUvksJglcljMEjksZommDYukr0raI+nBsnkvknSTpEezr0dn8yXpi1kTo/slra9m8Wa1lLJl+TqwacK8LcDNEbEGuDl7DvAmYE32uBT4l3zKNKu/acMSEb8Anp8w+63AVdn0VcDbyuZ/I0ruAJZKWplTrWZ1Ndsxy4qIeDqbfgZYkU0nNzKSdKmkbZK2Pfvss7Msw6x25jzAj1JTyhk3poyIKyNiQ0RsWL58+VzLMKu62YZl9/juVfZ1TzY/qZGRWSOabViuAy7Opi8GflA2/53ZUbGzgP1lu2tmDW3a/iySvg28HjhG0hPAJ4HLgWslXQLsAt6RvfxHwGagFxgE3l2Fms3qYtqwRMQFUyx64ySvDeB9cy3KbD7yGXyzRA6LWSKHxSyRw2KWyGExS1SV1t4z1dfXx4033ljvMszo6+ubctm8CEt3dzdnnnlmvcswo7u7e8pl8yIshUKBJUuW1LsMMwqFwpTLPGYxS+SwmCVyWMwSOSxmiRwWs0QOi1kih8UskcNilshhMUvksJglcljMEjksZokcFrNEDotZIofFLJHDYpYopZnR8ZJulbRD0kOSPpDNd0MjW1BStiyjwIcj4iTgLOB9kk7CDY1sgUlpZvR0RNyTTR8AHqbUc8UNjWxBmdGYRdJqYB2wlTk2NHIzI2s0yWGR1AN8F/hgRBxxv5jZNDRyMyNrNElhkdRGKSjfiojvZbPd0MgWlJSjYQK+AjwcEZ8tW+SGRragpNw3bCNwEfCApPuyeX+FGxrZApPSzOg2QFMsdkMjWzB8Bt8skcNilshhMUvksJglcljMEs2LlhM2NxFFisVBoEhLSyelc8iWN4elwY2O7mX37i+wf/91FIuDdHefwYoVH6Wz81RK55MtLw5LAxsb28+uXZeyb9/3Gb80b2iol/7+OzjxxO/Q1bWurvU1G49ZGtjzz1/Dvn0/YOI1rMPDj/PUU5+iWByuT2FNymFpUMPFIo/v/xVQnHT5wYP3EXGwtkU1OYelQfUXi2wbHJnycxGjtBJTXqVks+GwNKiRCG7kjRyk83eWFRE3T7HMZs9haVCjEWyLdVzBe+inm/FP341S4BbO4RtcRNG/3lz5aFiDGolghALXcD4PcTKv5TY6GWQ7p/IrXssxLK13iU3HYWlQoxEUIxijle2cxnZOq3dJTc/b6QY1EsFYvYtYYByWBrV7dJT+sanjsqqtjXafwc+Vw9KghorFiluWowoFCjWrZmFwWJpUm+Rrw3LmsDSpdsmnJHPmsDSpNocldw5Lk/KWJX8OS4Oa7n653rLkL+WOlB2S7pS0PevP8uls/sslbc36sFwjqT2bvyh73pstX13ln2FBGo7pby3tAX6+UrYsQ8A5EXEqcBqwKbst62eAz0XEK4AXgEuy118CvJDN/1z2OsvZUHHyS/OtelL6s0RE9GdP27JHAOcA38nmT+zPMt635TvAG+U/cbkbStiyWL5S76JfyO5zvAe4CXgM2BcRo9lLynuwHO7Pki3fDyzLsWYjbTfM8pUUlogYi4jTKLWPOANYO9cVu5nR3Hg3rPZmdDQsIvYBtwJnU2p/N37VcnkPlsP9WbLlS4DnJnkvNzOag+cqXBcGpctdLF8pR8OWS1qaTXcC51LqK3krcF72son9Wcb7tpwH3JLdWd9y9JvhyjejOGHRohpVsnCkfJ5lJXCVpAKlcF0bEddL2gFcLenvgHspNTwi+/pNSb3A88D5VajbpuErjvOX0p/lfkpNVyfOf5zS+GXi/EPA23OpzmZtUYvPN+fN/6JNapG3LLlzWJqUw5I/h6UBpRwv8Zglfw5LgxqbJjAtDkvuHJYGFPgMfj04LA3IYakPh6UBBaVbIVltOSwNKCjdRd9qy2FpQGMRHKgQFmUPy5fD0oBGIipeG9bT0sLKNveVzJvD0qAqjVhaJTp9uUvufGPwBraIQ2xgG6fwAIfo4HbO5lHW0EKBNp9nyZ3D0qB66OfjfIZN/IQ2RgC4iG/yz7yXX+ntDksVOCwNSAQX8G3O5YcUynpKHkUff8GXGORVtOqUOlbYnLxj24A6OMQmfnxEUKB0BKybATbrBm9ZqsBhaUAiaM92vX53GbTGkM/wV4HD0oDGaGdPy5pJj4gVEb8cPZH7BgdrXlezc1gaUGtLO9u7P8gzHHtEYAK4g7O4njfx8wMHki7lt3Qe4DegFoljF5/Flucu5918jbU8zDCLuI2NXMW76OMotg4MMBRBh8cuuXFYGtS6ri6eLJzKlrHL6WKQIi0M0E1kOws7Dh1iz+goL21vr3OlzcO7YQ3q+PZ2Vre3M0obfSyhn8WHgwKwd3SUuwcG6lhh83FYGlR3SwtndndPubwI3NjX53FLjhyWBiWJNy1ZUvEXePvAAIO+lD83DksDW9/VxbLWqYedO4eH2TXNnSstXXJYsjvp3yvp+uy5mxnV2YtbWzm5s3PK5X1jY9zj8y25mcmW5QOU7nE8zs2M6qxdqjhuCeBnBw5Q9LglF6n9WVYBfwT8e/ZcuJlR3UnidT09VLpf/u0DAwx43JKL1C3L54GPweEr95Yxx2ZG7s+Sj3VdXRxTYdzy2NAQOw4erGFFzSul5cSbgT0RcXeeK3Z/lnwsKxRY39U15fKhCG4fGPAh5BykbFk2Am+RtBO4mtLu1xeYYzMjy0dbSwvnHnVUxdf898AAlVsfWYqUBqyXRcSqiFhNqdfKLRFxIW5mNG+c0d1d8RqwbQMD9E3TKcymN5fzLB8HPpQ1LVrGkc2MlmXzPwRsmVuJNp21HR0cW+FuLk+OjPDwoUM1rKg5zehCyoj4GfCzbNrNjOaJowsFTu/qYucUJyCHI7itv5/XdHfjA5Oz5zP4TaBFmnbccmNfn2/5OkcOS5PY2NPD4gr3CvufQ4d4dnR0yuU2PYelSbysvb1ih+LdIyM85HHLnDgsTaKnpYXTK5xvGaN06YsPTM6ew9JE3rB4ccXlt/X3+64vc+CwNAllF1UuKUx9pdgDBw9OecTMpuewNJGXtLXxygrjlr6xMbYPDnpXbJYclibS1dJScVesCPyyv7/iHfhtag5Lk9nY01PxTPMv+/s55C3LrDgsTUQS67q6WFrhkv3Hh4bYOTRUw6qah8PSZI5ta+OUCh81HigW2eZxy6w4LE2mFXhdT8+Uy4vADX19+LOTM+ewNKEzu7srtpy4Z3CQfn/UeMYcliYjidO7unhJhUv2/3doiF5f+jJjDksTWtbayqsrjFuGI/hFf7/HLTPksDShFuCcCudbgtKlL94RmxmHpQlJ4jU9PXRWGLdsP3iQF/xR4xlxWJrUiYsWcdyEdhMCOiVO7ujgDxYv9m7YDLk/S5M6ulDg1Z2d7Boe5tjWVtZ3dfGGxYvZ2NPDmo4OFre04A8Yz4zD0qQE/M3Klbxn+XJO6ehgeVsbBfBn8OfAYWlS45e+WH48ZjFL5LCYJXJYzBI5LGaJHBazRA6LWSLNh7O4kg4Aj9S5jGOAvQt4/a6h5GURMWnDoPlynuWRiNhQzwIkbatnDfVev2uYnnfDzBI5LGaJ5ktYrqx3AdS/hnqvH1xDRfNigG/WCObLlsVs3qt7WCRtkvSIpF5JVes/KemrkvZIerBs3osk3STp0ezr0dl8SfpiVtP9ktbnsP7jJd0qaYekhyR9oJY1SOqQdKek7dn6P53Nf7mkrdl6rpHUns1flD3vzZavnuM/QXktBUn3Srq+XjXMSkTU7QEUgMeAE4B2YDtwUpXW9TpgPfBg2bx/BLZk01uAz2TTm4EfU/pYyFnA1hzWvxJYn00vBn4NnFSrGrL36cmm24Ct2fteC5yfzb8C+PNs+r3AFdn0+cA1Of4uPgT8B3B99rzmNcyq7rquHM4Gbih7fhlwWRXXt3pCWB4BVmbTKymd7wH4V+CCyV6XYy0/AM6tRw1AF3APcCalE4CtE38fwA3A2dl0a/Y65bDuVcDNwDnA9VmIa1rDbB/13g07Dvht2fMnsnm1siIins6mnwFW1KKubHdiHaW/7jWrIdv9uQ/YA9xEaau+LyLGm02Wr+Pw+rPl+ym1cJ+rzwMfg8M3l1lWhxpmpd5hmTei9Oer6ocGJfUA3wU+GBF9tawhIsYi4jRKf93PANZWa12TkfRmYE9E3F3L9eal3mF5Eji+7PmqbF6t7Ja0EiD7uqeadUlqoxSUb0XE9+pRA0BE7ANupbTLs1TS+GVP5es4vP5s+RLguTmueiPwFkk7gasp7Yp9ocY1zFq9w3IXsCY7GtJOaRB3XQ3Xfx1wcTZ9MaVxxPj8d2ZHpM4C9pftKs2KSneK+ArwcER8ttY1SFouaWk23UlpvPQwpdCcN8X6x+s6D7gl2/LNWkRcFhGrImI1pd/1LRFxYS1rmJN6DZbKBnybKR0Zegz46yqu59vA08AIpf3iSyjt/94MPAr8FHhR9loBX85qegDYkMP6X0tpF+t+4L7ssblWNQCvBu7N1v8g8Ils/gnAnUAv8J/Aomx+R/a8N1t+Qs6/j9fz/0fD6lLDTB8+g2+WqN67YWYNw2ExS+SwmCVyWMwSOSxmiRwWs0QOi1kih8Us0f8BivrJfUoR4J0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 216x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 导入gym库\n",
    "import gym\n",
    "\n",
    "# 创建环境，指定渲染模式为rgb_array，如果是在IDE中可以改为'human'\n",
    "env = gym.make('Acrobot-v1', render_mode='rgb_array')\n",
    "# 重置环境\n",
    "env.reset()\n",
    "# 创建GymHelper\n",
    "gym_helper = GymHelper(env)\n",
    "\n",
    "# 循环N次\n",
    "for i in range(20):\n",
    "    gym_helper.render(title = str(i)) # 渲染环境\n",
    "    action = env.action_space.sample() # 从动作空间中随机选取一个动作\n",
    "    observation, reward, terminated, truncated, info = env.step(action) # 执行动作\n",
    "    if terminated or truncated: # 如果游戏结束，则结束循环\n",
    "        break\n",
    "\n",
    "# 游戏结束\n",
    "gym_helper.render(title = \"Finished\")\n",
    "# 关闭环境\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a692f9c6",
   "metadata": {},
   "source": [
    "### SAC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a4c19fec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入必要的库\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import numpy as np\n",
    "import sys\n",
    "import time\n",
    "import random\n",
    "import collections\n",
    "from tqdm import * # 用于显示进度条\n",
    "\n",
    "# 策略模型，给定状态生成各个动作的概率\n",
    "class PolicyModel(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim):\n",
    "        super(PolicyModel, self).__init__()\n",
    "        \n",
    "        # 使用全连接层构建一个简单的神经网络，ReLU作为激活函数\n",
    "        # 最后加一个Softmax层，使得输出可以看作是概率分布\n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(input_dim, 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, output_dim),\n",
    "            nn.Softmax(dim = 1)\n",
    "        )\n",
    "\n",
    "    # 定义前向传播，输出动作概率分布\n",
    "    def forward(self, x):\n",
    "        action_prob = self.fc(x)\n",
    "        return action_prob\n",
    "\n",
    "# Q网络模型，给定状态和动作样本对估计Q值\n",
    "class QValueModel(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim):\n",
    "        super(QValueModel, self).__init__()\n",
    "        \n",
    "        # 网络结构和策略模型类似，输出维度为动作空间的维度\n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(input_dim, 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, output_dim)\n",
    "        )\n",
    "\n",
    "    # 定义前向传播，输出Q值估计\n",
    "    def forward(self, x):\n",
    "        value = self.fc(x)\n",
    "        return value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5cddccd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 经验回放缓冲区\n",
    "class ReplayBuffer:\n",
    "    # 构造函数，max_size是缓冲区的最大容量\n",
    "    def __init__(self, max_size):\n",
    "        self.max_size = max_size\n",
    "        self.buffer = collections.deque(maxlen = self.max_size)  # 用collections的队列存储，先进先出\n",
    "\n",
    "    # 添加experience（五元组）到缓冲区\n",
    "    def add(self, state, action, reward, next_state, done):\n",
    "        experience = (state, action, reward, next_state, done)\n",
    "        self.buffer.append(experience)\n",
    "    \n",
    "    # 取出buffer中的所有数据\n",
    "    def return_all(self):\n",
    "        batch = list(self.buffer)\n",
    "        state, action, reward, next_state, done = zip(*batch)\n",
    "        return np.array(state), np.array(action), np.array(reward), np.array(next_state), np.array(done)\n",
    "\n",
    "    # 从buffer中随机采样，数量为batch_size\n",
    "    def sample(self, batch_size):\n",
    "        if batch_size > len(self.buffer):\n",
    "            return self.return_all()\n",
    "        batch = random.sample(self.buffer, batch_size)\n",
    "        state, action, reward, next_state, done = zip(*batch)\n",
    "        return np.array(state), np.array(action), np.array(reward), np.array(next_state), np.array(done)\n",
    "    \n",
    "    # 返回缓冲区数据数量\n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4ccdcbf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# SAC算法\n",
    "class SAC:\n",
    "    # 构造函数，参数包含环境，学习率，折扣因子，更新目标网络参数\n",
    "    def __init__(self, env, learning_rate=0.001, gamma=0.99, rho=0.01):\n",
    "        self.env = env\n",
    "        self.gamma = gamma\n",
    "        self.rho = rho\n",
    "#         self.replay_buffer = ReplayBuffer(max_size=buffer_size)\n",
    "        \n",
    "        # 设置一个较大的目标熵值，取负值转为最小化问题\n",
    "        self.target_entropy = -np.log2(env.action_space.n)\n",
    "        \n",
    "        # 判断可用的设备是 CPU 还是 GPU\n",
    "        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "        \n",
    "        # 创建模型，并将模型移动到指定设备上\n",
    "        # 包括策略网络、两个Q值网络和它们对应的目标网络\n",
    "        self.actor = PolicyModel(env.observation_space.shape[0], env.action_space.n).to(self.device)\n",
    "        self.q1 = QValueModel(env.observation_space.shape[0], env.action_space.n).to(self.device)\n",
    "        self.q2 = QValueModel(env.observation_space.shape[0], env.action_space.n).to(self.device)\n",
    "        self.target_q1 = QValueModel(env.observation_space.shape[0], env.action_space.n).to(self.device)\n",
    "        self.target_q2 = QValueModel(env.observation_space.shape[0], env.action_space.n).to(self.device)\n",
    "        \n",
    "        # 将Q值网络的参数复制到对应的目标网络中，以确保初始的Q值网络和目标网络参数相同\n",
    "        for param, target_param in zip(self.q1.parameters(), self.target_q1.parameters()):\n",
    "            target_param.data.copy_(param)\n",
    "        for param, target_param in zip(self.q2.parameters(), self.target_q2.parameters()):\n",
    "            target_param.data.copy_(param)\n",
    "        \n",
    "        # 定义Adam优化器\n",
    "        self.optimizer_actor = torch.optim.Adam(self.actor.parameters(), lr=learning_rate)\n",
    "        self.optimizer_q1 = torch.optim.Adam(self.q1.parameters(), lr=learning_rate)\n",
    "        self.optimizer_q2 = torch.optim.Adam(self.q2.parameters(), lr=learning_rate)\n",
    "        \n",
    "        # alpha作为可学习参数，学习其对数值可确保alpha=exp(log_alpha)>0\n",
    "        self.log_alpha = torch.tensor([0.0], device=self.device, requires_grad=True)\n",
    "        self.optimizer_log_alpha = torch.optim.Adam([self.log_alpha], lr=learning_rate)\n",
    "\n",
    "    # 使用模型生成动作概率分布并采样\n",
    "    def choose_action(self, state):\n",
    "        # 将状态转换为tensor输入模型\n",
    "        state = torch.FloatTensor(np.array([state])).to(self.device)\n",
    "        with torch.no_grad():\n",
    "            action_prob = self.actor(state)\n",
    "        \n",
    "        # 生成分布后采样返回动作\n",
    "        c = torch.distributions.Categorical(action_prob)\n",
    "        action = c.sample()\n",
    "        return action.item()\n",
    "    \n",
    "    # 模型更新\n",
    "    def update(self, buffer):\n",
    "        # 取出数据，并将其转换为numpy数组\n",
    "        # 然后进一步转换为tensor，并将数据转移到指定计算资源设备上\n",
    "        states, actions, rewards, next_states, dones = buffer\n",
    "#         states, actions, rewards, next_states, dones = self.replay_buffer.sample(batch_size)\n",
    "        states = torch.FloatTensor(np.array(states)).to(self.device)\n",
    "        actions = torch.tensor(np.array(actions)).view(-1, 1).to(self.device)\n",
    "        rewards = torch.FloatTensor(np.array(rewards)).view(-1, 1).to(self.device)\n",
    "        next_states = torch.FloatTensor(np.array(next_states)).to(self.device)\n",
    "        dones = torch.FloatTensor(np.array(dones)).view(-1, 1).to(self.device)\n",
    "        \n",
    "        ##### Q网络更新 #####\n",
    "        # 计算下一个状态的动作概率和对数\n",
    "        next_action_prob = self.actor(next_states)\n",
    "        log_next_prob = torch.log(next_action_prob + 1e-9) # 避免对零取对数\n",
    "        \n",
    "        # 计算目标Q值\n",
    "        target_q1 = self.target_q1(next_states)\n",
    "        target_q2 = self.target_q2(next_states)\n",
    "        target_q_min = torch.min(target_q1, target_q2)\n",
    "        min_q_next_target = next_action_prob * (target_q_min - torch.exp(self.log_alpha) * log_next_prob)\n",
    "        min_q_next_target = torch.sum(min_q_next_target, dim = 1, keepdim = True)\n",
    "        \n",
    "        # 计算TD目标\n",
    "        td_target = rewards + (1 - dones) * self.gamma * min_q_next_target\n",
    "        \n",
    "        # 计算Q网络的loss\n",
    "        q1 = self.q1(states)\n",
    "        q2 = self.q2(states)\n",
    "        q1_loss = F.mse_loss(q1.gather(1, actions), td_target.detach()).mean()\n",
    "        q2_loss = F.mse_loss(q2.gather(1, actions), td_target.detach()).mean()\n",
    "        \n",
    "        # 梯度清零、反向传播、更新参数\n",
    "        self.optimizer_q1.zero_grad()\n",
    "        self.optimizer_q2.zero_grad()\n",
    "        q1_loss.backward()\n",
    "        q2_loss.backward()\n",
    "        self.optimizer_q1.step()\n",
    "        self.optimizer_q2.step()\n",
    "        \n",
    "        ##### actor网络更新 #####\n",
    "        # 计算当前状态的动作概率\n",
    "        action_prob = self.actor(states)\n",
    "        log_prob = torch.log(action_prob + 1e-9)\n",
    "        \n",
    "        # 计算Q值和actor网络的loss\n",
    "        q1 = self.q1(states)\n",
    "        q2 = self.q2(states)\n",
    "        inside_term = torch.exp(self.log_alpha) * log_prob - torch.min(q1, q2)\n",
    "        actor_loss = torch.sum(action_prob * inside_term, dim = 1, keepdim = True).mean()\n",
    "        \n",
    "        # 梯度清零、反向传播、更新参数\n",
    "        self.optimizer_actor.zero_grad()\n",
    "        actor_loss.backward()\n",
    "        self.optimizer_actor.step()\n",
    "        \n",
    "        ##### alpha参数更新 #####\n",
    "        # 计算alpha值的loss\n",
    "        inside_term = -torch.sum(action_prob * log_prob, dim = 1, keepdim = True) - self.target_entropy\n",
    "        alpha_loss = (torch.exp(self.log_alpha) * inside_term.detach()).mean()\n",
    "        # 梯度清零、反向传播、更新参数\n",
    "        self.optimizer_log_alpha.zero_grad()\n",
    "        alpha_loss.backward()\n",
    "        self.optimizer_log_alpha.step()\n",
    "        \n",
    "        ##### target网络更新 #####\n",
    "        for param, target_param in zip(self.q1.parameters(), self.target_q1.parameters()):\n",
    "            target_param.data.copy_(param.data * self.rho + target_param.data * (1 - self.rho))\n",
    "        for param, target_param in zip(self.q2.parameters(), self.target_q2.parameters()):\n",
    "            target_param.data.copy_(param.data * self.rho + target_param.data * (1 - self.rho))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5dd0b57",
   "metadata": {},
   "source": [
    "### MBPO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a45e44a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义计算设备\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# 定义一个用于构造集成模型的全连接类\n",
    "class EnsembleFC(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim, ensemble_size):\n",
    "        super(EnsembleFC, self).__init__()\n",
    "        \n",
    "        # 定义输入维度、输出维度和集成模型的数量\n",
    "        self.input_dim = input_dim\n",
    "        self.output_dim = output_dim\n",
    "        self.ensemble_size = ensemble_size\n",
    "        \n",
    "        # 初始化权重矩阵和偏置向量。为每个集成模型都初始化一个权重矩阵和一个偏置向量。\n",
    "        # 权重矩阵的大小是 (ensemble_size, input_dim, output_dim)\n",
    "        # 偏置向量的大小是 (ensemble_size, 1, output_dim)\n",
    "        # nn.Parameter可以自动被添加到模型的参数列表中，使得在优化时可以被自动更新\n",
    "        self.weight = nn.Parameter(torch.Tensor(ensemble_size, input_dim, output_dim).to(device))\n",
    "        self.bias = nn.Parameter(torch.Tensor(ensemble_size, output_dim).to(device))\n",
    "\n",
    "    # 定义前向传播\n",
    "    def forward(self, x):\n",
    "        # 使用矩阵乘法将输入x和权重矩阵相乘\n",
    "        w_times_x = torch.bmm(x, self.weight)\n",
    "        # 将矩阵乘法的结果与偏置向量（需匹配维度）相加，得到最终的输出\n",
    "        return torch.add(w_times_x, self.bias[:, None, :])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4d59a8a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义一个集成模型类\n",
    "class EnsembleModel(nn.Module):\n",
    "    # 构造函数，参数包含状态空间维度、动作空间维度、模型个数、奖励维度、隐藏层大小、学习率\n",
    "    def __init__(self, state_size, action_size, ensemble_size, reward_size=1, hidden_size=128, learning_rate=0.001):\n",
    "        super(EnsembleModel, self).__init__()\n",
    "        \n",
    "        # 定义隐藏层大小、输入维度、输出维度、运算设备\n",
    "        self.hidden_size = hidden_size\n",
    "        self.input_dim = state_size + action_size\n",
    "        self.output_dim = state_size + reward_size\n",
    "        self.device = device\n",
    "        \n",
    "        # 使用EnsembleFC分别创建4个全连接层\n",
    "        # 最后的输出为均值和方差，所以是预期输出维度乘以2\n",
    "        self.nn1 = EnsembleFC(self.input_dim, hidden_size, ensemble_size)\n",
    "        self.nn2 = EnsembleFC(hidden_size, hidden_size * 2, ensemble_size)\n",
    "        self.nn3 = EnsembleFC(hidden_size * 2, hidden_size, ensemble_size)\n",
    "        self.nn4 = EnsembleFC(hidden_size, self.output_dim * 2, ensemble_size)\n",
    "        \n",
    "        # 初始化log方差的最大最小值\n",
    "        self.max_logvar = nn.Parameter((torch.ones((1, self.output_dim)).float() / 2).to(self.device), requires_grad=False)\n",
    "        self.min_logvar = nn.Parameter((torch.zeros((1, self.output_dim)).float()).to(self.device), requires_grad=False)\n",
    "        \n",
    "        # 定义Adam优化器\n",
    "        self.optimizer = torch.optim.Adam(self.parameters(), lr=learning_rate)\n",
    "        \n",
    "        # 定义初始化权重的方法，并对模型的所有权重进行初始化\n",
    "        def init_weights(m):\n",
    "            if type(m) == nn.Linear or isinstance(m, EnsembleFC):\n",
    "                torch.nn.init.kaiming_normal_(m.weight, nonlinearity='relu')\n",
    "                torch.nn.init.constant_(m.bias, 0)\n",
    "        self.apply(init_weights)\n",
    "    \n",
    "    # 定义前向传播\n",
    "    def forward(self, x, ret_log_var=False):\n",
    "        # 通过全连接网络进行特征提取和处理，激活函数使用ReLU函数\n",
    "        x = torch.relu(self.nn1(x))\n",
    "        x = torch.relu(self.nn2(x))\n",
    "        x = torch.relu(self.nn3(x))\n",
    "        x = self.nn4(x)\n",
    "        \n",
    "        # 计算预测的均值\n",
    "        mean = x[:, :, :self.output_dim]\n",
    "\n",
    "        # 计算预测的log方差，并通过softplus函数进行变换保证log方差能平滑的落在最大最小值之间\n",
    "        logvar = self.max_logvar - F.softplus(self.max_logvar - x[:, :, self.output_dim:]) # 映射到(-inf, max]\n",
    "        logvar = self.min_logvar + F.softplus(logvar - self.min_logvar) # 映射到[min, max]\n",
    "        \n",
    "        # 根据标志参数返回不同的结果，差异在于返回方差的对数还是原始值\n",
    "        if ret_log_var:\n",
    "            return mean, logvar\n",
    "        else:\n",
    "            return mean, torch.exp(logvar)\n",
    "    \n",
    "    # 定义损失函数，接收预测的均值和log方差以及真实的标签\n",
    "    def loss(self, mean, logvar, labels):\n",
    "        # 计算方差的倒数，用来作为损失函数的一个权重，增加方差较小的样本的重要性\n",
    "        inv_var = torch.exp(-logvar)\n",
    "        \n",
    "        # 计算预测均值与真值的loss\n",
    "        mean_loss = torch.mean(torch.mean(torch.pow(mean - labels, 2) * inv_var, dim=-1), dim=-1)\n",
    "        # 方差loss\n",
    "        var_loss = torch.mean(torch.mean(logvar, dim=-1), dim=-1)\n",
    "        # 求和后返回\n",
    "        total_loss = torch.sum(mean_loss) + torch.sum(var_loss)\n",
    "        return total_loss, mean_loss\n",
    "    \n",
    "    # 定义训练函数，输入计算后的损失函数\n",
    "    def train(self, loss):\n",
    "        # 清空梯度、反向传播、更新参数即可\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        self.optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8f244aae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义模型类\n",
    "class EnsembleDynamicsModel:\n",
    "    # 构造函数，参数包含状态的维度、动作的维度、网络数量\n",
    "    def __init__(self, state_size, action_size, network_size=5):\n",
    "        # 初始化参数、模型\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.network_size = network_size\n",
    "        self.ensemble_model = EnsembleModel(state_size, action_size, ensemble_size=network_size)\n",
    "        self.device = device\n",
    "    \n",
    "    # 定义训练函数\n",
    "    def train(self, inputs, labels, batch_size=256, holdout_ratio=0.2, max_iter=10):\n",
    "        # 根据holdout_ratio计算验证集所需数量\n",
    "        num_holdout = int(inputs.shape[0] * holdout_ratio)\n",
    "        # 将输入数据和标签打乱\n",
    "        permutation = np.random.permutation(inputs.shape[0])\n",
    "        inputs, labels = inputs[permutation], labels[permutation]\n",
    "        \n",
    "        # 划分训练集和验证集\n",
    "        train_inputs, train_labels = inputs[num_holdout:], labels[num_holdout:]\n",
    "        holdout_inputs, holdout_labels = inputs[:num_holdout], labels[:num_holdout]\n",
    "        \n",
    "        # 转换数据类型和形状以适应模型的输入维度\n",
    "        holdout_inputs = torch.from_numpy(holdout_inputs).float().to(self.device)\n",
    "        holdout_labels = torch.from_numpy(holdout_labels).float().to(self.device)\n",
    "        holdout_inputs = holdout_inputs[None, :, :].repeat([self.network_size, 1, 1])\n",
    "        holdout_labels = holdout_labels[None, :, :].repeat([self.network_size, 1, 1])\n",
    "        \n",
    "        # 尝试训练max_iter个epoch\n",
    "        for epoch in range(max_iter):\n",
    "            # 对每个模型生成一个随机排列的索引\n",
    "            train_idx = np.vstack([np.random.permutation(train_inputs.shape[0]) for _ in range(self.network_size)])\n",
    "            # 循环开始分批训练\n",
    "            for start_pos in range(0, train_inputs.shape[0], batch_size):\n",
    "                # 获取当前批次的数据和标签\n",
    "                idx = train_idx[:, start_pos: start_pos + batch_size]\n",
    "                train_input = torch.from_numpy(train_inputs[idx]).float().to(device)\n",
    "                train_label = torch.from_numpy(train_labels[idx]).float().to(device)\n",
    "                \n",
    "                # 前向传播、计算损失、更新参数\n",
    "                mean, logvar = self.ensemble_model(train_input, ret_log_var=True)\n",
    "                loss, _ = self.ensemble_model.loss(mean, logvar, train_label)\n",
    "                self.ensemble_model.train(loss)\n",
    "    \n",
    "    # 预测方法，将输入数据变换后分别输入到所有模型中，返回所有模型的预测结果\n",
    "    def predict(self, inputs):\n",
    "        # 输入数据变换\n",
    "        x = np.tile(inputs, (self.network_size, 1, 1))\n",
    "        x = torch.FloatTensor(x).to(self.device)\n",
    "        # 输入模型后得到预测的均值和方差，并返回结果\n",
    "        ensemble_mean, ensemble_var = self.ensemble_model(x)\n",
    "        return ensemble_mean.detach().cpu().numpy(), ensemble_var.detach().cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "049cb96f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义模拟环境类\n",
    "class FakeEnv:\n",
    "    # 构造函数，参数为定义的集成模型\n",
    "    def __init__(self, model):\n",
    "        self.model = model\n",
    "    \n",
    "    # 模拟环境的step操作\n",
    "    def step(self, state, action):\n",
    "        # 将状态和动作拼接成输入\n",
    "        inputs = np.concatenate((state, [action]), axis=-1)\n",
    "        # 使用模型预测下一个状态和奖励的均值和方差\n",
    "        ensemble_model_means, ensemble_model_vars = self.model.predict(inputs)\n",
    "        # 将下一状态的均值与当前状态相加，模型预测的是状态的变化量\n",
    "        ensemble_model_means[:,:,1:] += state\n",
    "        # 计算标准差\n",
    "        ensemble_model_stds = np.sqrt(ensemble_model_vars)\n",
    "        # 通过随机生成分布得到模型的输出值\n",
    "        ensemble_samples = ensemble_model_means + np.random.normal(size=ensemble_model_means.shape) * ensemble_model_stds\n",
    "        \n",
    "        # 从模型集合中随机选择模型\n",
    "        num_models, batch_size, _ = ensemble_model_means.shape\n",
    "        model_idx = np.random.choice([i for i in range(self.model.network_size)], size=batch_size)\n",
    "        batch_idx = np.arange(0, batch_size)\n",
    "        \n",
    "        # 从随机选择的模型的预测值中取出对应的奖励值和下一个状态，并返回结果\n",
    "        samples = ensemble_samples[model_idx, batch_idx]\n",
    "        rewards, next_states = samples[:,:1][0][0], samples[:,1:][0]\n",
    "        return rewards, next_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "139b0953",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义MBPO类\n",
    "class MBPO:\n",
    "    # 构造函数，参数包含环境，真实环境、模拟环境回放缓冲区大小，模拟环境数据生成次数，真实环境采样比例\n",
    "    def __init__(self, env, buffer_size=20000, rollout_size=5000, rollout_length=1, real_ratio=0.5):\n",
    "\n",
    "        # 创建SAC agent\n",
    "        self.env = env\n",
    "        self.agent = SAC(env)\n",
    "        \n",
    "        # 获取环境状态和动作维度\n",
    "        state_dim = env.observation_space.shape[0]\n",
    "        action_dim = 1 # action维度需根据实际情况调整，这里设为1\n",
    "        \n",
    "        # 使用集成模型创建模拟环境\n",
    "        ensemble_model = EnsembleDynamicsModel(state_dim, action_dim, network_size=5)\n",
    "        self.fake_env = FakeEnv(ensemble_model)\n",
    "        \n",
    "        # 创建两个回放缓冲区：一个用于真实环境，一个用于模型环境\n",
    "        self.env_pool = ReplayBuffer(buffer_size)\n",
    "        self.model_pool = ReplayBuffer(rollout_size)\n",
    "        \n",
    "        # 保存其他参数\n",
    "        self.rollout_length = rollout_length\n",
    "        self.rollout_batch_size = rollout_size\n",
    "        self.real_ratio = real_ratio\n",
    "    \n",
    "    # 在真实环境中进行探索以获取新的数据\n",
    "    def explore(self):\n",
    "        # 重置环境\n",
    "        state, _ = self.env.reset()\n",
    "        done, episode_return = False, 0\n",
    "        # 循环进行每一步操作\n",
    "        while not done:\n",
    "            # 根据当前状态选择动作\n",
    "            action = self.agent.choose_action(state)\n",
    "            # 执行动作，获取新的信息\n",
    "            next_state, reward, terminated, truncated, info = self.env.step(action)\n",
    "            # 判断是否达到终止状态\n",
    "            done = terminated or truncated\n",
    "            \n",
    "            # 将这个五元组加到buffer中\n",
    "            self.env_pool.add(state, action, reward, next_state, done)\n",
    "            # 累计奖励\n",
    "            episode_return += reward\n",
    "            # 更新当前状态\n",
    "            state = next_state\n",
    "        return episode_return\n",
    "    \n",
    "    # 模拟虚拟环境以收集模型数据\n",
    "    def rollout_model(self):\n",
    "        # 从真实环境缓冲区中抽取样本\n",
    "        states, _, _, _, _ = self.env_pool.sample(self.rollout_batch_size)\n",
    "        for state in states:\n",
    "            for i in range(self.rollout_length):\n",
    "                # 通过agent选择动作并在模拟环境中执行动作来收集数据\n",
    "                action = self.agent.choose_action(state)\n",
    "                reward, next_state = self.fake_env.step(state, action)\n",
    "                \n",
    "                # 将结果添加到模拟环境回放缓冲区中\n",
    "                self.model_pool.add(state, action, reward, next_state, False)\n",
    "                # 更新状态\n",
    "                state = next_state\n",
    "    \n",
    "    # 训练模拟环境\n",
    "    def train_model(self):\n",
    "        # 取出真实环境缓冲区中的所有数据\n",
    "        states, actions, rewards, next_states, dones = self.env_pool.return_all()\n",
    "        # 状态和动作合并为输入\n",
    "        inputs = np.concatenate(((states, np.reshape(actions, (actions.shape[0], -1)))), axis=-1)\n",
    "        # 奖励和状态的差值合并为标签\n",
    "        labels = np.concatenate((np.reshape(rewards, (rewards.shape[0], -1)), next_states - states), axis=-1)\n",
    "        # 训练环境模型\n",
    "        self.fake_env.model.train(inputs, labels)\n",
    "    \n",
    "    # 基于真实和模拟环境数据上执行训练来更新agent\n",
    "    def update_agent(self, policy_train_batch_size=4096):\n",
    "        # 计算分别使用多少数据进行训练\n",
    "        env_batch_size = int(policy_train_batch_size * self.real_ratio)\n",
    "        model_batch_size = policy_train_batch_size - env_batch_size\n",
    "        # 循环迭代\n",
    "        for epoch in range(10):\n",
    "            # 从真实环境缓冲区中抽取样本\n",
    "            env_states, env_actions, env_rewards, env_next_states, env_dones = self.env_pool.sample(env_batch_size)\n",
    "            if len(self.model_pool) > 0:\n",
    "                # 从模拟环境缓冲区中抽取样本（如果有的话）\n",
    "                model_states, model_actions, model_rewards, model_next_states, model_dones = self.model_pool.sample(model_batch_size)\n",
    "                # 与真实环境数据合并\n",
    "                states = np.concatenate((env_states, model_states), axis=0)\n",
    "                actions = np.concatenate((env_actions, model_actions), axis=0)\n",
    "                rewards = np.concatenate((env_rewards, model_rewards), axis=0)\n",
    "                next_states = np.concatenate((env_next_states, model_next_states), axis=0)\n",
    "                dones = np.concatenate((env_dones, model_dones), axis=0)\n",
    "            else:\n",
    "                # 如果模拟环境缓冲区中没有数据，则只使用真实环境的数据\n",
    "                states, actions, rewards, next_states, dones = env_states, env_actions, env_rewards, env_next_states, env_dones\n",
    "            \n",
    "            # 将数据传入agent更新模型\n",
    "            buffer = (states, actions, rewards, next_states, dones)\n",
    "            self.agent.update(buffer)\n",
    "    \n",
    "    # 主训练循环\n",
    "    def train(self, num_episode=10):\n",
    "        # 定义保存每个回合奖励的列表\n",
    "        return_list = [] \n",
    "        # 训练刚开始时进行初始探索\n",
    "        explore_return = self.explore()\n",
    "\n",
    "        # 开始循环，tqdm用于显示进度条并评估任务时间开销\n",
    "        for episode in tqdm(range(num_episode), file=sys.stdout):\n",
    "            # 重置环境并获取初始状态\n",
    "            state, _ = env.reset()\n",
    "            done, episode_reward = False, 0\n",
    "            \n",
    "            # 每轮循环开始时训练环境模型并生成虚拟数据\n",
    "            self.train_model()\n",
    "            self.rollout_model()\n",
    "            \n",
    "            # 循环进行每一步操作\n",
    "            while not done:\n",
    "                # 根据当前状态选择动作\n",
    "                action = self.agent.choose_action(state)\n",
    "                # 执行动作，获取新的信息\n",
    "                next_state, reward, terminated, truncated, info = self.env.step(action)\n",
    "                # 判断是否达到终止状态\n",
    "                done = terminated or truncated\n",
    "                \n",
    "                # 将这个五元组加到真实环境缓冲区中\n",
    "                self.env_pool.add(state, action, reward, next_state, done)\n",
    "                # 累计奖励\n",
    "                episode_reward += reward\n",
    "                # 更新当前状态\n",
    "                state = next_state\n",
    "\n",
    "                # 更新agent\n",
    "                self.update_agent()\n",
    "            \n",
    "            # 记录当前回合奖励值\n",
    "            return_list.append(episode_reward)\n",
    "            \n",
    "            # 打印中间值\n",
    "            if episode % (num_episode // 10) == 0:\n",
    "                tqdm.write(\"Episode \" + str(episode) + \": \" + str(episode_reward))\n",
    "        return return_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "58331160",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 0: -500.0                     \n",
      "Episode 1: -246.0                             \n",
      "Episode 2: -191.0                             \n",
      "Episode 3: -193.0                             \n",
      "Episode 4: -257.0                             \n",
      "Episode 5: -204.0                             \n",
      "Episode 6: -430.0                             \n",
      "Episode 7: -225.0                              \n",
      "Episode 8: -169.0                              \n",
      "Episode 9: -110.0                              \n",
      "100%|██████████| 10/10 [13:07<00:00, 78.80s/it]\n"
     ]
    }
   ],
   "source": [
    "mbpo = MBPO(env)\n",
    "return_list = mbpo.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f5783c64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAGxCAYAAACXwjeMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABc1ElEQVR4nO3de1yUdd4//tfMMCdOw2GUM4LnAx4BFW2X3DztunZ3713dRppkWW66HbCT2m1Kqdtm7vfONs1D6pZldWu11a/7Rs3NWk+AoKApeUAQBBRlhuMMzFy/P2BGJ1ABZ+aaw+v5eMxjZbiAN6w5Lz6f9/V5SwRBEEBERETkpqRiF0BERER0JxhmiIiIyK0xzBAREZFbY5ghIiIit8YwQ0RERG6NYYaIiIjcGsMMERERuTWGGSIiInJrDDNERETk1hhmiIg64Z///CckEgn++c9/il0KEf0CwwwRERG5NYYZInKahoYGsUu4pcbGRrFLIKJuYJghIodYtmwZJBIJjh49ivvvvx/BwcHo06cPBEHAu+++ixEjRkCtViM4OBj3338/zp07Z/3Yv/3tb5BKpaiqqrI+99Zbb0EikWD+/PnW58xmM4KDg7Fw4ULrc8uXL8eYMWMQEhKCwMBAjBo1Cps3b8YvZ+rGxcXh97//PXbt2oWRI0dCpVJh+fLlAIBTp05h6tSp8PX1hVarxbx581BbW+uoHxUR3SEfsQsgIs/2hz/8ATNmzMC8efNQX1+PJ598Elu3bsXTTz+NN954A1evXkVmZibGjRuHY8eOISwsDBMnToQgCNi7dy8eeughAMCePXugVquxe/du6+fOyclBTU0NJk6caH2uuLgYTz75JGJjYwEAhw4dwp/+9CeUlZVh6dKlNrUdPXoUP/30E1555RXEx8fDz88PlZWVSE1NhVwux7vvvouwsDBs374dCxYscMJPi4i6RSAicoBXX31VACAsXbrU+tzBgwcFAMJbb71lc21paamgVquFF1980fpcdHS0MGfOHEEQBMFgMAh+fn7CSy+9JAAQLly4IAiCIKxYsUKQy+VCXV1dhzWYTCahublZyMzMFEJDQwWz2Wx9X69evQSZTCacPn3a5mNeeuklQSKRCPn5+TbPT5o0SQAg7Nu3r+s/DCJyKG4zEZFD/cd//If1z19//TUkEglmzpyJlpYW6yM8PBzDhw+3uVPonnvuwZ49ewAABw4cQENDAzIyMqDVaq2rM3v27EFKSgr8/PysH/fdd99h4sSJ0Gg0kMlkkMvlWLp0Kaqrq222rQBg2LBh6N+/v81z+/btw5AhQzB8+HCb59PS0uzy8yAi+2OYISKHioiIsP65srISgiAgLCwMcrnc5nHo0CFcuXLFeu3EiRNRUlKCn3/+GXv27MHIkSPRs2dP/OY3v8GePXvQ2NiIAwcO2GwxHTlyBJMnTwYAbNy4Ef/617+QnZ2NJUuWAGjf4HtjbRbV1dUIDw9v93xHzxGRa2DPDBE5lEQisf5Zq9VCIpHghx9+gFKpbHftjc/dc889AFpXX3bv3o1JkyZZn3/llVewf/9+GAwGmzCzY8cOyOVyfP3111CpVNbnv/jii9vWZhEaGoqKiop2z3f0HBG5Bq7MEJHT/P73v4cgCCgrK0NSUlK7x9ChQ63XRkREYPDgwdi5cydyc3OtYWbSpEm4fPky1qxZg8DAQCQnJ1s/RiKRwMfHBzKZzPpcY2MjPvjgg07XOGHCBJw4cQLHjh2zef6jjz7q7rdNRA7GMENETjN+/Hg88cQTePTRR/Hiiy/i66+/xr59+/DRRx/hqaeewrp162yuv+eee7B3714oFAqMHz8eABAfH4/4+HhkZWXh7rvvho/P9QXmadOmoa6uDmlpadi9ezd27NiBX/3qVx2uAt3Ms88+C61Wi2nTpmHr1q349ttvMXPmTJw6dco+PwQisjuGGSJyqvfeew/vvPMO9u/fjxkzZmDatGlYunQp6uvrMXr0aJtrLVtId911l822keX5G7eYAOA3v/kN3n//fRQUFGD69OlYsmQJ7r//frz88sudri88PBzff/89Bg8ejD/+8Y+YOXMmVCoV3nnnne5+y0TkYBJB+MVJUkRERERuhCszRERE5NYYZoiIiMitMcwQERGRW2OYISIiIrfGMENERERujWGGiIiI3JpXjDMwm80oLy9HQEBAh8eXExERkesRBAG1tbWIjIyEVHrz9RevCDPl5eWIiYkRuwwiIiLqhtLSUkRHR9/0/V4RZgICAgC0/jACAwNFroaIiIg6Q6/XIyYmxvo6fjNeEWYsW0uBgYEMM0RERG7mdi0ibAAmIiIit+bQMLNixQqMGzcOvr6+CAoK6vCakpISTJ8+HX5+ftBqtXj66adhNBptrikoKEBqairUajWioqKQmZkJjpQiIiIiwMHbTEajEQ888ABSUlKwefPmdu83mUyYNm0aevTogR9//BHV1dWYPXs2BEHA2rVrAbTul02aNAkTJkxAdnY2ioqKkJ6eDj8/PyxcuNCR5RMREZEbcGiYWb58OQBg69atHb4/KysLJ0+eRGlpKSIjIwEAb731FtLT07FixQoEBgZi+/btaGpqwtatW6FUKpGQkICioiKsWbMGGRkZvNWaiIjIy4naM3Pw4EEkJCRYgwwATJkyBQaDAbm5udZrUlNToVQqba4pLy9HcXFxh5/XYDBAr9fbPIiIiMgziRpmKioqEBYWZvNccHAwFAoFKioqbnqN5W3LNb+0atUqaDQa64NnzBAREXmuLoeZZcuWQSKR3PKRk5PT6c/X0TaRIAg2z//yGkvz7822mBYtWgSdTmd9lJaWdroeIiIici9d7plZsGABZsyYcctr4uLiOvW5wsPDcfjwYZvnrl27hubmZuvqS3h4eLsVmKqqKgBot2JjoVQqbbaliIiIyHN1OcxotVpotVq7fPGUlBSsWLECly5dQkREBIDWpmClUonExETrNYsXL4bRaIRCobBeExkZ2enQRERERJ7LoT0zJSUlyM/PR0lJCUwmE/Lz85Gfn4+6ujoAwOTJkzF48GDMmjULeXl52Lt3L55//nnMnTvXelJvWloalEol0tPTUVhYiM8//xwrV67knUxEREQEAJAIDjx9Lj09Hdu2bWv3/L59+3D33XcDaA08Tz31FL777juo1WqkpaVh9erVNttEBQUFmD9/Po4cOYLg4GDMmzcPS5cu7XSY0ev10Gg00Ol0HGdARETkJjr7+u3QMOMqGGaIiIjcT2dfvzmbiYiIiLqtqLIWaRsPoaq2SbQaGGaIiIioWyp0TZj9/hEcOFuN17/+SbQ6GGaIiIioy/RNzUjfcgSXdE3o3cMPy+8dIlotDDNERETUJcYWM+Z9kItTFbXoEaDEtkdHI9hPIVo9DDNERETUaWazgBf+5xgOnK2Gn0KGLenJiAnxFbUmhhkiIiLqtDf+7xS+zC+Hj1SCdTMTkRClEbskhhkiIiLqnG0HivHe9+cAAH/+j2H4df8eIlfUimGGiIiIbut/Cy9h2VcnAADPT+6P+xOjRa7oOoYZIiIiuqXs4qt4ekc+BAF4eEws5k/oK3ZJNhhmiIiI6KbOVNXi8W05MLaYMXFQGDL/LcHlZiMyzBAREVGHqvRNmP1+NnSNzRgZG4S1D42ETOpaQQZgmCEiIqIO1DY1I31LNspqGhGv9cPm2clQK2Ril9UhhhkiIiKyYWwx46ntR3Hykh5afwW2PToaISIeinc7DDNERERkJQgCXt51HD/8fAW+ChneT09GbKi4h+LdDsMMERERWa3OOo1dR8sgk0rwt4dHYVh0kNgl3RbDDBEREQEAPjx0AX/bdxYAsOrfh2LCgJ4iV9Q5DDNERESErBMVWPplIQDg2Yn98GByjMgVdR7DDBERkZfLvXANT+/Ig1kAZiTH4Jl7+oldUpcwzBAREXmxc5fr8Pi2bDQ1mzFhQA+8fp/rHYp3OwwzREREXqqqtgmztxzBtYZmDI/W4G8Pj4KPzP2igftVTERERHes3tCCx7bmoPRqI3qF+mJzejJ8FT5il9UtDDNEREReptnUeiheQZkOIX6th+Jp/ZVil9VtDDNEREReRBAELN5VgO+LLkMll2Lz7CTEaf3ELuuOMMwQERF5kb/u+Rmf5V6EVAL8LW0URsYGi13SHWOYISIi8hIfHynB23t/BgC8ft9Q3DMoTOSK7INhhoiIyAt8d6oSr3zReijen37TF2ljYkWuyH4YZoiIiDxcfmkN5m/Pg8ks4P7EaGRM6i92SXbFMENEROTBiq/UY87WbDQ2m/Dr/j2w6g9D3e5QvNthmCEiIvJQV+oMmL3lCK7WG5EQFYh3Hx4FuRseinc7nvcdERERERqMLXhsazYuVDcgOliN99OT4a90z0PxbodhhoiIyMO0mMxY8FEejl3UIdhXjm1zRqNngErsshyGYYaIiMiDCIKA//qyEN+dqoLSR4pNs5PRp4e/2GU5lEPDzIoVKzBu3Dj4+voiKCiow2skEkm7x/r1622uKSgoQGpqKtRqNaKiopCZmQlBEBxZOhERkVta+90ZfHykFFIJ8PZDI5HYy/0Pxbsdh26eGY1GPPDAA0hJScHmzZtvet2WLVswdepU69sajcb6Z71ej0mTJmHChAnIzs5GUVER0tPT4efnh4ULFzqyfCIiIrfyaU4p1uwuAgAsv3cIpgwJF7ki53BomFm+fDkAYOvWrbe8LigoCOHhHf/At2/fjqamJmzduhVKpRIJCQkoKirCmjVrkJGR4XG3lxEREXXHP09XYdGuAgDAH+/ug1kpceIW5EQu0TOzYMECaLVaJCcnY/369TCbzdb3HTx4EKmpqVAqr0/znDJlCsrLy1FcXNzh5zMYDNDr9TYPIiIiT1VwUYenth+FySzg30dG4cUpA8QuyalEDzOvvfYaPvvsM+zZswczZszAwoULsXLlSuv7KyoqEBZmOzvC8nZFRUWHn3PVqlXQaDTWR0xMjOO+ASIiIhGVVDfg0a1H0GA04a6+WrzxH8O8bteiy2Fm2bJlHTbt3vjIycnp9Od75ZVXkJKSghEjRmDhwoXIzMzEm2++aXPNL/9PsTT/3uz/rEWLFkGn01kfpaWlXfwuiYiIXN/VeiNmbzmCK3VGDIoIxLqZo6DwEX2dwum63DOzYMECzJgx45bXxMXFdbcejB07Fnq9HpWVlQgLC0N4eHi7FZiqqioAaLdiY6FUKm22pYiIiDxNo9GEx7Zl4/yVekQFqbH10WQEqORilyWKLocZrVYLrVbriFoAAHl5eVCpVNZbuVNSUrB48WIYjUYoFAoAQFZWFiIjI+8oNBEREbkrk1nA0zvykFdSA41ajm1zkhEW6LmH4t2OQ+9mKikpwdWrV1FSUgKTyYT8/HwAQN++feHv74+vvvoKFRUVSElJgVqtxr59+7BkyRI88cQT1pWVtLQ0LF++HOnp6Vi8eDF+/vlnrFy5EkuXLvW6PUEiIiJBEPDqPwqx+2QlFD5SbJqdhL49A8QuS1QODTNLly7Ftm3brG+PHDkSALBv3z7cfffdkMvlePfdd5GRkQGz2YzevXsjMzMT8+fPt36MRqPB7t27MX/+fCQlJSE4OBgZGRnIyMhwZOlEREQu6d1/nsWHh0ogkQD//Z8jkBwXInZJopMIXnCUrl6vh0ajgU6nQ2BgoNjlEBERdcuuoxeR8ekxAMCr0wfj0fHxIlfkWJ19/fa+lmciIiI39MPPl/Hi/xwHADzx694eH2S6gmGGiIjIxZ0o1+GPHx5Fi1nA9OGReHnqQLFLcikMM0RERC7s4rUGpG/JRp2hBSm9Q7H6gWGQSnkDzI0YZoiIiFxUTYMRs98/gsu1BgwIC8D6WYlQ+sjELsvlMMwQERG5oKZmEx7floOzl+sRoVFh65xkaNTeeSje7TDMEBERuRiTWcCzO/KRc+EaAlQ+2ProaERo1GKX5bIYZoiIiFyIIAh47euT+N8TFVDIpNgwKwkDwr37ULzbYZghIiJyIRv2n8PWA8UAgLceHI6UPqHiFuQGGGaIiIhcxJf5ZVj17SkAwCvTBmH68EiRK3IPDDNEREQu4LtTlVjYdrrvnPHxePxXvUWuyH0wzBAREYns0Llq66F4942IxCvTBoldklthmCEiIhJRwUUdHt+WA0OLGRMH9cSbDwznoXhdxDBDREQkkjNVtXjk/cOoM7RgbO8QvJM2CnIZX5q7ij8xIiIiEZRebcDMTUdwraEZw6I12DQ7GSo5T/ftDoYZIiIiJ6uqbcKszYdRoW9Cv57+2ProaPgrfcQuy20xzBARETmRrqEZj2w+guLqBkQHq/HBY2MQ4qcQuyy3xjBDRETkJA3GFjy69QhOVdSiR4ASHz42BuEaldhluT2GGSIiIicwtJjw5Ae5OFpSA41ajg8eG404rZ/YZXkEhhkiIiIHazGZ8eyOfPzw8xX4KmTY8mgyBoYHil2Wx2CYISIiciCzWcCiXQX4tvD64MhRscFil+VRGGaIiIgcRBAErPj/fsJnuRchlQBvPzQSd/XTil2Wx2GYISIicpC1353B5h/PAwD+cv9wTE0IF7kiz8QwQ0RE5ABb/3Uea3YXAQCW/n4w7k+MFrkiz8UwQ0REZGc7cy9i2VcnAQDP3NMPc+6KF7kiz8YwQ0REZEdZJyrw4s7jAIBHx8fh2Yn9RK7I8zHMEBER2cmBM1ew4KM8mMwC7k+Mxn9NGwyJhBOwHY1hhoiIyA7yS2vw+N9zYDSZMWVIGP78h6GQShlknIFTrYgc4EqdAa9/fRLluiao5TKo5FKo5TKoFTKo5LK259r+V9H6vzdeZ3nOck3rc1IoZFL+lkfkgk5X1CJ9yxE0GE24q68Wbz80Ej4yrhc4C8MMkZ3pGpoxa/MR/HRJb/fPLZXANgwpbghAluCjkEHl88vgJLV5+/rHS9uFqyBfOQMTUReUVDdg1ubDqGloxsjYILw3KxFKH5nYZXkVhhkiO6oztGD2ltYgo/VXYsm0gTCbgcZmE5raHo3NJjQazR08d/3PTc1t7ze2vt1iFgAAZgFoMJrQYDQ57HsY3zcUm2cnQyXnP8ZEt1Opb8LDmw+hqtaAAWEB2JKeDD8lX1qdjT9xIjtpNJrw2NZs5JfWIMhXjg8fH2232SvNJvP1oHNDEGq0PveLEPSLcGT73C8+3mgJVWYYTWb860w1Xvyf4/jvGSO4QkN0C9fqjZi1+TBKrzaiV6gvPnhsNIJ8FWKX5ZUYZojswNhixh+35+Lw+avwV/rg73PsF2QAQC6TQi6TIkAlt9vn7MjBs9WYtfkw/nGsHIMiAvHHu/s49OsRuas6QwvSt2ajqLIOYYFKfPjYGPQMVIldltdyWHdScXExHnvsMcTHx0OtVqNPnz549dVXYTQaba4rKSnB9OnT4efnB61Wi6effrrdNQUFBUhNTYVarUZUVBQyMzMhCIKjSifqkhaTGc/syMM/T1+GSi7F++nJGBYdJHZZ3ZLSJxSv3jsEAPCX/zuFvT9VilwRketpajZh7rYcHCutQbCvHB8+NgYxIb5il+XVHLYyc+rUKZjNZrz33nvo27cvCgsLMXfuXNTX12P16tUAAJPJhGnTpqFHjx748ccfUV1djdmzZ0MQBKxduxYAoNfrMWnSJEyYMAHZ2dkoKipCeno6/Pz8sHDhQkeVT9QpZrOAF//nuM003NHxIWKXdUdmje2Fny7p8dHhEjyzIx9fzB+Hvj0DxC6LyCW0mMz408d5OHiuGn4KGbY+Ohr9wvjfh9gkghOXON58802sW7cO586dAwB8++23+P3vf4/S0lJERkYCAHbs2IH09HRUVVUhMDAQ69atw6JFi1BZWQmlUgkA+POf/4y1a9fi4sWLndrT1+v10Gg00Ol0CAy039I/eTdBEPBfXxbiw0MlkEklWPfwKEwe4hlD5IwtZszcdBhHiq8iLtQXX86/Cxpfx25xEbk6s1nA858dw668Mih8pNj26Gik9AkVuyyP1tnXb6feBK/T6RAScv231oMHDyIhIcEaZABgypQpMBgMyM3NtV6TmppqDTKWa8rLy1FcXNzh1zEYDNDr9TYPInsSBAGrvj2FDw+VQCIB1jw43GOCDAAofKRYN3MUooLUKK5uwIKPj6LFZBa7LCLRCIKAzK9PYldeGWRSCd5NG8Ug40KcFmbOnj2LtWvXYt68edbnKioqEBYWZnNdcHAwFAoFKioqbnqN5W3LNb+0atUqaDQa6yMmJsae3woR3t57Bhv2t64wrvz3ofi3EVEiV2R/of5KbHwkCWq5DD/8fAWrvj0ldklEovnr7iJsPVAMiQR464HhmDg47PYfRE7T5TCzbNkySCSSWz5ycnJsPqa8vBxTp07FAw88gMcff9zmfR1tEwmCYPP8L6+x7IzdbItp0aJF0Ol01kdpaWlXv02im9r0wzn8dU8RAOC/fj8YD42OFbkixxkcGYi3HhwOANj843l8lsP/lsj7bPrhHN7+7gwAIPPeIbhvpOf98uLuutwAvGDBAsyYMeOW18TFxVn/XF5ejgkTJiAlJQUbNmywuS48PByHDx+2ee7atWtobm62rr6Eh4e3W4GpqqoCgHYrNhZKpdJmW4rIXj46XILXv/kJALBwUn88dle8yBU53u+GRuDpe/rh7b0/Y8nnhejdwx+JvYLFLsut/X8Fl9BgNOH+xGixS6Hb+DS71Prf/AtTBmBWSpy4BVGHuhxmtFottFptp64tKyvDhAkTkJiYiC1btkAqtV0ISklJwYoVK3Dp0iVEREQAALKysqBUKpGYmGi9ZvHixTAajVAoFNZrIiMjbUITkaN9kVeGJV8UAACeTO2NBb/pK3JFzvPsPf1wukKP/ztRiXkf5uIfC8YjQqMWuyy3tOmHc9YXx+S4YPQK9RO5IrqZbwsu4eVdxwEAT/y6N57iuUsuy2E9M+Xl5bj77rsRExOD1atX4/Lly6ioqLBZZZk8eTIGDx6MWbNmIS8vD3v37sXzzz+PuXPnWruW09LSoFQqkZ6ejsLCQnz++edYuXIlMjIyeDopOc3/FlZg4WfHIAitty6/PHWgV/39k0olWPPgCAwIC8DlWgOe+HsumpodN1LBU73/43lrkAGA7OJrIlZDt7K/6DKe3pEHswDMSI7Bot9613/z7sZhYSYrKwtnzpzBd999h+joaERERFgfFjKZDN988w1UKhXGjx+PBx98EPfdd5/1HBoA0Gg02L17Ny5evIikpCQ89dRTyMjIQEZGhqNKJ7LxfdFl/OnjozCZBfzHqGgsv3eIV/6j5qf0wabZSQj2laOgTIeXdh7n4ZVdsO1AMTK/PgkAiA5uXdXKKb4qZkl0E7kXruLJD3LRbBIwbWgEVvz7UK/8b96dOPWcGbHwnBnqrsPnqjF7yxE0NZvxu6HheHvGSPjInHqigcs5cPYKZm0+ApNZwMu/HYh5qVx6v50PDl3Af31RCAD44919MCo2GHP/noO+Pf2xJyNV5OroRj9d0uM/3zsIfVMLUvv3wMZHkqDw8e7/5sXkkufMELmTY6U1eGxbDpqazZgwoAf+338yyADAuD5avDp9MADgjf89he9OceTBrXx0uMQaZJ78dW+8OGWAtYH6TFUdahqMt/pwcqLzV+oxa/MR6JtakNQrGOtnJjLIuAn+v0TUgZ8u6fHI+0dQZ2hBSu9QrOM/ajZmje2Fh0bHQhCAZz7Ox5mqWrFLckmfZJdg8eetTeOP3xWPl9v6LkL8FOjTo7XxN/cC+2ZcwSVdI2ZuOowrdQYMjgjE5vRkqBUyscuiTuK/zkS/cO5yHWZtPgxdYzNGxgZh4+wkqOT8R+1GEokEy+8dgtFxIag1tGDu33Oha2gWuyyX8llOKV7e1RpkHh0fhyXTBtn0XST1aj0NnU3A4quuM2DmpsMoq2lEvNYP2+aMhkbN8R3uhGGG6AalVxvw8KbDuFJnxOCIQGxNHw1/pcPmsbo1hY8U77aNPDh/pZ4jD26wM/ciXtx5HIIAzE7phaW/H9yugTQxrnWrKfcCm4DFVNvUjPQt2Th7uR6RGhU+fHwMegTwnDJ3wzBD1KZS34SZmw/jkq4JfXr44YPHRnO44m1o/ZXY8EiideTBnznyAF/kleH5/2m9jX/m2Fgsu8ndb8lxrSszxy7qYGjhbe5iaGo24bFtOSgo0yHUT4EPHh+DqCCen+SOGGaIAFytN2LmpsO4UN2AmBA1tj8+FqH+/O2sM4ZEarD6gdaRB5t+PI//yb0ockXi+TK/DBmf5kMQgIdGxyLz3oSb3tIbF+qLUD8FjC1mFJbpnFwpNZvMeGr7URw5fxUBSh9smzMafXr4i10WdRPDDHk9XWMzZm0+jJ+r6hAeqMJHj49FuEYldlluZdqwCPyp7UTkxbsKcLTE+/pAvj5ejuc+yYdZAP4zKQYr7kuAVHrzs0kkEon1rqYc9s04lcksYOGnx/DdqSqo5FJsTk9GQpRG7LLoDjDMkFdrMLZgztZsnCjXI9RPgQ8fH4OYEF+xy3JLz03sj0mDw2A0mfHkB7mo0DWJXZLTfFtwCc/saA0y9ydGY9Ufht4yyFgktfXN5PCOJqcRBAFLvyzEP46Vw0cqwbqZiRgdHyJ2WXSHGGbIazU1mzD37znIvXANgSoffPDYGPTtyWXm7pJKJfjrf45A/zB/XK414MkPcrxi5MH/FlbgTx/nwWQW8IeRUXjjP4Z1KsgAQFJb30zuhWs8TdlJ3vy/09h+uAQSCfDX/xyBCQN6il0S2QHDDHmlZpMZ87cfxb/OVMNPIcO2OaMxOJKnQ98pf6UPNj2SjCBfOY5d1OFlDx95sPtkJRZ8dBQtZgH3jYjEmw8Mh6yTQQYAEiI1UPpIcbXeiHNX6h1YKQHA+u/P4t1/ngUArPz3oZg+PFLkisheGGbI65jMAp77JB97T1VB6SPFptnJGBkbLHZZHiM21Bfvpo2CTCrBF/nl2LD/nNglOcTenyrx1PZctJgFTB8eidVdDDJA6+3tw6ODAAC57JtxqI8Ol1jvtlv024F4aHSsyBWRPTHMkFcxmwW8vPM4vj5+CXKZBOtnJSKlT6jYZXmccX21WPr71pEHf/7fU9h3qkrkiuxr3+kq/PHDo9ZBhH99cHi3R11c75vheTOO8tWxciz5ovUAw6fu7oMnOU/M4zDMkNcQBAGZX5/EZ7kXIZUAb88Yyf1yB3okpRdmJMdAEICnP87Dmao6sUuyi++LLuPJD3JhNJnx24Rw/L8ZI+5oZpc1zHBlxiH2narCc5/kW8/9eWHKALFLIgdgmCGvsTrrNLYeKG798wPD8duhEeIW5OEkEgky/y0ByXHBqDW04Im/57j9yIMff76CJ/6eA2OLGZMHh+Hth0ZCfofDR0e1bXGeu1KP6jqDPcqkNtnFVzHvw9atwHuHR97y3B9ybwwz5BX+tu8M/ravtfHvtfsS8IdR0SJX5B0UPlKsm5mISI0K567U4087Wu/6cUcHzlzBY9uyYWgxY+KgMLyTNuqOgwwABPkq0K/tLjoOnbSvV788AUOLGb8Z2BNvPTi803eZkfthmCGPt+Vf5/Hm/50GACz+3UDMGttL5Iq8S+vIgySo5FLsL7qMP3/7k9glddmhc9WY0xZkfjOwJ/728Ei7TlG/8RZtso/qOgNOXtIDAP5y/zC7BE9yXfx/lzzap9mlWP7VSQDAM/f0wxO/ZuOfGBKiro882PjDeex0o5EHR85fxaNbstHUbMbdA3pg3cxRUPrYd4p6UttJwNnFbAK2l0PnWn+WA8MDoOVoEo/HMEMe66tj5Xhp13EAwON3xePZif1Ersi7/X5YJBZMaB15sOjzAuS5wciDnOKrSN9yBI3NJvyqnxbrZybaPcgA15uAC8v0XnHQoDMcPHcFADC2N+9W9AYMM+SR9pystN7BkDYmFkumDWLjnwvImNQfEweFwdjSOvKgUu+6Iw9yL1zD7PePoMFowl19tdj4SBJUcvsHGQCIDfFFjwAljCYzCjh00i4OnK0GAB694CUYZsjj/PjzFTx1w6msr/8b72BwFa0jD4ajf5g/qmoNeOLvrjnyIK+kNcjUG00Y1yfUoUEGaL3zi1tN9lOpb8K5y/WQSICx8Qwz3oBhhjxKTvFVzG27dXbKkDCsfoB3MLiaAJUcGx9Jso48WLSrwKVGHhwrrcEjm4+gztCCsb1DsGl2EtQKxwUZC8sEbZ4EfOcOnWtdlRkSGQiNr1zkasgZGGbIYxSW6fDolmw0Npvw6/498PZDI+/oMDNynF6hfvhb28iDz/PKsPEH1xh5UFimw6zNh1FraMHouBBsnp0MX4WPU7629Y6mkmswu+nt667iwJm2LSb2y3gN/ktPHqGosvb6i1B8CN5zUKMm2c/4vlr817RBAIA/f3sK+06LO/KgsEyHhzcdhr6pBUm9gvH+o8nwUzonyACtqwgquRQ1Dc04d8UzTksWy8G2lZlxfbQiV0LOwjBDbq/4Sj1mbjqMaw3NGB6twWYnbQvQnZs9Lg7/mRQDc9vIg7OXxXkRP1mux8zNh6FrbMao2CBseTQZ/k4MMgAgl0kxIiYIAJDNraZuu3itASVXGyCTSpAcHyJ2OeQkDDPk1sprGvHwpsOoqjVgYHgAts0ZjQAV98jdhUQiQeZ9Q5DYKxi1TS2Yuy0Hukbnjjw4VaHHw5sOoaahGcNjgrBVxL9DSb1aX3w5p6n7DrbdxTQ0SuP0QEriYZght1VV24SHNx1GWU0jemv98MFjYxDkqxC7LOoipY8M62cmIqJt5MHTHztv5EFRZS0e3ti6qjcsWoO/zxmNQBHDsOW8mVxO0O6261tM7JfxJgwz5Jau1Rsxa9MRnL9Sj6ggNT58fAx6BPCUT3fVI0DZdvuzFN8XXcYb/3vK4V/zTFUt0jYeQnW9EQlRgfhgzhho1OKu6o3qFQyJBCiubsDlWg6d7CpBEHCI58t4JYYZcju1Tc2YveUITlfWomeAEh/NHYPIILXYZdEdSojS4M37W0cebNh/DruOOm7kwdnLdXho42FcqTNicEQgPnxsjEvcwhuokmNAWAAArs50x4XqBpTrmiCXSaxbduQdGGbIrTQaTXhsaw6OX9Qh2FeO7Y+PQa9QP7HLIjuZPjwS8ye0zs96eVcB8ktr7P41zl2uw0MbDuFyW5/V9sdda3vSct4M+2a6zrLFNDImmDcBeBmGGXIbhhYTnvggB0eKryJA6YMPHhuDfm2/xZLnWDhpACYO6gljixlP/D3HriMPiq/U46GNh1BVa8CAsNYgE+znOkEGAJLbzpvJ4QTtLrM0/47lFpPXYZght9BsMuNPH+Xhh5+vQC2XYeucZCREacQuixygdeTBCPTr2Tby4INcu4w8KKluwEMbD6FSb0C/nv7YPncMQl1wmrJlZaawTIdGo+uNenBVgiBY5zGx+df7MMyQyzObBbzw2TFknayEwkeKTbOTkMj9cI9mGXmgUctxrLQGi+9w5EHp1dYgc0nXhD49/PDR3LHQumCQAYDoYDXCApVoMQs4drFG7HLcxtnLdbhSZ4DSR4qRsUFil0NO5rAwU1xcjMceewzx8fFQq9Xo06cPXn31VRiNRpvrJBJJu8f69ettrikoKEBqairUajWioqKQmZnpUrNcyLH2nqrCF/nl8JFKsO7hURjfl6d6eoM47fWRB7vyyrDph/Pd+jwXrzVgxoZD1lv4P5471qXvfGsdOtk22oBbTZ1mWZVJ7BXM07+9kMNOFDp16hTMZjPee+899O3bF4WFhZg7dy7q6+uxevVqm2u3bNmCqVOnWt/WaK5vH+j1ekyaNAkTJkxAdnY2ioqKkJ6eDj8/PyxcuNBR5ZMLsUwRfiApBvcMChO5GnKmu/ppseR3g5D59Ums+vYn9Avzx90Denb648tqGvHQxtYgE6/1w8dPjEXPQJUDK7aPpLhgfFNwiRO0u+Agt5i8msPCzNSpU20CSu/evXH69GmsW7euXZgJCgpCeHh4h59n+/btaGpqwtatW6FUKpGQkICioiKsWbMGGRkZkEg4EdnTHWu7o2Vk21Hv5F0eHR+HUxV6fJpzEX/6OA9fzB+PPj38b/txl3SNSNt4CKVXG9Er1Bcfzx2LMDcIMsD1k4CPXmgdOsnJ77dmNgvWSdk8X8Y7ObVnRqfTISSkfa/DggULoNVqkZycjPXr18NsNlvfd/DgQaSmpkKpvL4sPGXKFJSXl6O4uLjDr2MwGKDX620e5J7MZgGFZToAwLAYNvx6I4lEgtfuS7g+8uDvOdA33XrkQaW+CWkbD+NCdQNiQtT4eO5YhGvcI8gAwKCIAPgqZNA3teDnKg6dvJ1TFbW41tAMX4UMw6KDxC6HROC0MHP27FmsXbsW8+bNs3n+tddew2effYY9e/ZgxowZWLhwIVauXGl9f0VFBcLCbLcWLG9XVFR0+LVWrVoFjUZjfcTExNj5uyFnOXelDvVGE9RyGfp24rdx8kxKHxnWzRzVOvLg8q1HHlTpm/DQhkM4f6Ue0cGtQcbdDlX0kV1vYs3h4Xm3ZTlfJjkuBHIZ72vxRl3+f33ZsmUdNu3e+MjJybH5mPLyckydOhUPPPAAHn/8cZv3vfLKK0hJScGIESOwcOFCZGZm4s0337S55pdbSZbm35ttMS1atAg6nc76KC0t7eq3SS7iWGnrqkxCVCB8+I+UV+sZoMKGWUlQ+kjxz9OX8ZcORh5U1TbhoY2HcK5tzMXHc8ciOthXhGrvXCKHTnbaQY4w8Hpd7plZsGABZsyYcctr4uLirH8uLy/HhAkTkJKSgg0bNtz2848dOxZ6vR6VlZUICwtDeHh4uxWYqqoqAGi3YmOhVCpttqXIfR1vuzV1aFSQqHWQaxgarcFf7h+GZ3bk47395zAwIgD/PjIaAHClzoCHNx7G2cv1iNSo8PHcsYgJcc8gAwBJlpOAuTJzSyazgMPn28JMb4YZb9XlMKPVaqHVdu7W2LKyMkyYMAGJiYnYsmULpNLb/2adl5cHlUqFoKAgAEBKSgoWL14Mo9EIhaL1pM6srCxERkbahCbyTMfb+mWGs1+G2vzbiCicqqjFun+exUs7CxCv9UdMsBoPbzyMn6vqEB6owsdPjEVsqPsGGQAYGRsEqQQovdqISn2T2zQvO9uJch1qm1oQoPLBkMhAscshkThs3b68vBx33303YmJisHr1aly+fBkVFRU2qyxfffUVNm7ciMLCQpw9exabNm3CkiVL8MQTT1hXVtLS0qBUKpGeno7CwkJ8/vnnWLlyJe9k8gLNJjNOlrc2b7Opj270/OQBuGdg68iDJz/IwcObDlsHj378xFiPmNcVoJJjYHjrizO3mm7OssU0Jj6EW9FezGG3ZmdlZeHMmTM4c+YMoqOjbd5n6XmRy+V49913kZGRAbPZjN69eyMzMxPz58+3XqvRaLB7927Mnz8fSUlJCA4ORkZGBjIyMhxVOrmI0xW1MLSYEaDyQZyb/5ZN9iWTSvD/ZozAv797AGeq6lCpN6BHW5CJ17p/kLFIigvGyUt65Fy4imnDIsQuxyVZDssbyy0mr+awMJOeno709PRbXvPLs2huZujQodi/f7+dKiN3cfxi2y3Z0RquwlE7ASo5Nj2ShAfeOwiZRIIPHx/dqfNn3Elir2D8/eAFngR8E80ms/VgwXF9eDK4N3NYmCG6UwVlNQC4xUQ3F6f1w/4XJkAmlUDh43lbDJYJ2ifK9WgwtsBXwX+yb3T8og4NRhOCfeUYGB4gdjkkIs/7r588huW27OHRbP6lm1MrZB4ZZAAgMkiNSI0KJrOA/JIasctxOQfPXgHQusXEU5K9m2f+C0Bur6nZhNOVtQC4MkPeLbFtdSaHW03tHOQIA2rDMEMu6US5HiazAK2/AhFudAw9kb1dP2+GYeZGhhaT9S4vni9DDDPkkgraDssbFh3E5l/yaklxrWHm6IVrNx3h4I3ySmpgaDFD669E356e1fhNXccwQy7pxjuZiLzZwPBA+Ct9UGdowemKWrHLcRk3jjDgLzzEMEMu6Vjbysxw9suQl5NJJdahk7kcbWBlDTPcYiIwzJALqm1qxrkr9QBaZ/EQebukXmwCvlGj0YS80tafxTg2/xIYZsgFFZbpIQhAVJAaWn8ODCWy9M1wrEGr3AvX0GwSEKFRoRdPBycwzJALOm5t/uWqDBEAjIgJgkwqQVlNIy7pGsUuR3QH2s6XSenNfhlqxTBDLud682+QuIUQuQg/pQ8GRbSecMvVGZ4vQ+0xzJDLOcaVGaJ2rH0zxd7dBFxnaLH+wsMwQxYMM+RSrtYbcfFa6zJ6QhTDDJGFtW/Gy5uAs89fhcksICZEjehg9stQK4YZcimWfpneWj9o1HJxiyFyIZaVmZ8u6VFnaBG5GvFYtpjG9eaUbLqOYYZcCg/LI+pYuEaF6GA1zAK8eujkjYflEVkwzJBLsazMDGXzL1E7ljlN2V7aN6NraEZhOftlqD2GGXIplpWZ4VyZIWrHMkE710v7Zg6fr4YgAL17+CEskANo6TqGGXIZFbomVNUaIJNKMCSSYYbolywrM3kl19BiMotcjfMd4AgDugmGGXIZlluy+/X0h1ohE7cYIhfUPywAASof1BtNOOWFQycPWZp/+7D5l2wxzJDL4Mm/RLcmk0owKtYy2sC7+maq6wzWADe2d4jI1ZCrYZghl8GTf4luz7LV5G3nzRw61xreBoQFIJQz2+gXGGbIJQiCgIIyS/NvkLjFELmwpDjLScDXIAiCyNU4z8FzbfOYeBcTdYBhhlxCydUG1DQ0QyGTYkB4gNjlELmsETFB8JFKUKFvQlmN9wyd5PkydCsMM+QSjrVtMQ2KCIDCh38tiW5GrZBhSGQgAO+5RbtS34Szl+shkQBj4xlmqD2+apBLOF5aA4D9MkSdkdjr+laTN7DcxTQkMhAaX445ofYYZsglHC/jGAOizkqO866TgA/yfBm6DYYZEp3JLKDQ0vwbEyRuMURuILEtzJyurIW+qVnkahzvAPtl6DYYZkh0Zy/XocFogq9Chj49/MUuh8jl9QxQITbEF4IA5Hn40MmymkaUXG2ATCpBchzPl6GOMcyQ6I619cskRGogk0rELYbITSS1rc7kevhWk2WLaWiUBgEq9stQxxhmSHQF7Jch6rKktibgbA9vAj5wtvV8mXHcYqJbYJgh0Vluyx7GfhmiTrOszOSX1qDZQ4dOCoKAQ+yXoU5gmCFRGVvM+KlcDwAYFsWVGaLO6tvDH4EqHzQ2m/DTJb3Y5TjEheoGlOuaIJdJrCtRRB1xaJi59957ERsbC5VKhYiICMyaNQvl5eU215SUlGD69Onw8/ODVqvF008/DaPRaHNNQUEBUlNToVarERUVhczMTK86xtuTna6ohdFkhkYtR69QX7HLIXIbUqnEOtrAU7eaDradLzMyJhhqhUzkasiVOTTMTJgwAZ9++ilOnz6NnTt34uzZs7j//vut7zeZTJg2bRrq6+vx448/YseOHdi5cycWLlxovUav12PSpEmIjIxEdnY21q5di9WrV2PNmjWOLJ2c5HhZDYDWfhmJhM2/RF2R2DZ0MveCZzYBW5p/x3KLiW7Dx5Gf/LnnnrP+uVevXnj55Zdx3333obm5GXK5HFlZWTh58iRKS0sRGRkJAHjrrbeQnp6OFStWIDAwENu3b0dTUxO2bt0KpVKJhIQEFBUVYc2aNcjIyOALoJs7XsrmX6Lusk7Qbhs66Un/HgqCcP18GR6WR7fhtJ6Zq1evYvv27Rg3bhzk8tbb6w4ePIiEhARrkAGAKVOmwGAwIDc313pNamoqlEqlzTXl5eUoLi7u8GsZDAbo9XqbB7mmYxdrAABDo4JErYPIHQ2PCYJcJkFVrQEXr3nW0Mmzl+twpc4ApY8UI2ODxC6HXJzDw8xLL70EPz8/hIaGoqSkBF9++aX1fRUVFQgLC7O5Pjg4GAqFAhUVFTe9xvK25ZpfWrVqFTQajfURExNjz2+J7KTRaMLPVXUAgOExXJkh6iqVXIaEtsZ5TxttYNliSuwVDJWc/TJ0a10OM8uWLYNEIrnlIycnx3r9Cy+8gLy8PGRlZUEmk+GRRx6xad7taFn0l8ulv7zG8vE3W1JdtGgRdDqd9VFaWtrVb5Oc4OQlHUxmAT0ClAgPVIldDpFbsm41edgEbW4xUVd0uWdmwYIFmDFjxi2viYuLs/5Zq9VCq9Wif//+GDRoEGJiYnDo0CGkpKQgPDwchw8ftvnYa9euobm52br6Eh4e3m4FpqqqCgDardhYKJVKm20pck3H2vplhrP5l6jbEnuFYOMP55HrQXc0mc2CdVL2uL4MM3R7XQ4zlnDSHZYVFYPBAABISUnBihUrcOnSJURERAAAsrKyoFQqkZiYaL1m8eLFMBqNUCgU1msiIyNtQhO5n+PslyG6Y0k3DJ3UNTRD4+v+R/6frqzFtYZm+CpkGBYdJHY55AYc1jNz5MgRvPPOO8jPz8eFCxewb98+pKWloU+fPkhJSQEATJ48GYMHD8asWbOQl5eHvXv34vnnn8fcuXMRGBgIAEhLS4NSqUR6ejoKCwvx+eefY+XKlbyTyQMct578y34Zou7S+isRr/UDABwt8YzVGcsWU1JcCOQynu1Kt+ewvyVqtRq7du3CPffcgwEDBmDOnDlISEjA999/b90Ckslk+Oabb6BSqTB+/Hg8+OCDuO+++7B69Wrr59FoNNi9ezcuXryIpKQkPPXUU8jIyEBGRoajSicn0Dc149yVegDAcP7mRXRHEq19M57RBGxp/uU8Juosh50zM3ToUHz33Xe3vS42NhZff/31bT/X/v377VUauYDCtlWZ6GA1QvwUIldD5N6S44LxP7kXkeMBfTMms4DD59n8S13D9TsShXW4JA/LI7pjiW1zi/JLa2Bsce+hkyfKdahtakGA0gdDIgPFLofcBMMMicLS/MvmPqI716eHH4J95TC0mHGiXCd2OXfEssU0pncIfNgvQ53EvykkiuNcmSGyG4lEcsOcJvfearIMlxzLLSbqAoYZcrrqOgPKalqPXh8axTBDZA/XJ2i7bxNws8mMI+db6x/Xp3tHgJB3Ypghp7OsyvTu4YcAlfufiUHkCpJuWJm58ZR1d3L8og4NRhOCfeUYGB4gdjnkRhhmyOkswyV5SzaR/SREaaCQSXGlzogL1Q1il9MtB89eAQCMiQ+FVMpzxKjzGGbI6QrYL0Nkdyq5zPrflLvOaTrIEQbUTQwz5FSCINxwW3aQuMUQeZjEttEGOW7YN2NoMVnPyeH5MtRVDDPkVJd0TbhSZ4BMKsHgCJ4hQWRPSW3nzbjjykxeSQ0MLWZo/ZXo29Nf7HLIzTDMkFNZzpfpHxYAtUImbjFEHsZye/aZqjpcqzeKXE3XWM6XSekTyrl71GUMM+RUljuZhrNfhsjuQvwU6NOjdeiku503Y+mX4RYTdQfDDDnVcfbLEDmUO241NRpNyGub+J3C4ZLUDQwz5DSCINwwxoArM0SOYGkCznWjCdq5F66h2SQgQqNCXKiv2OWQG2KYIacprm6AvqkFCh8pBvBALCKHSG47CfjYRR0MLSaRq+mcg+daz5dJ6c1+GeoehhlyGsuqzOCIQMg5QI7IIeJCfRHqp4CxxYzCMvcYOnmgrfl3LLeYqJv4ikJOw+ZfIse7ceik5dwWV1ZnaLH+2zCOYYa6iWGGnMayMjOUzb9EDpVkOTzPDZqAs4uvwmQWEBOiRnQw+2WoexhmyClaTGYUlukBcGWGyNEsE7TdYeik5XyZcb05JZu6j2GGnOLs5Xo0Npvgp5Chdw+e7knkSAmRGih9pLhab8S5K/Vil3NLNx6WR9RdDDPkFJZJ2QlRGsg4DZfIoRQ+UutU+lwX7pvRNTSjsLy1X4Zhhu4Ewww5Bc+XIXKu630zrnvezOHz1RAEoHcPP4QFqsQuh9wYwww5BU/+JXIua5hx4ZUZjjAge2GYIYcztJjw0yVL82+QuMUQeYlRsa1h5tyVelTXGUSupmPslyF7YZghhztdUYtmk4AgXzliQtRil0PkFYJ8FejXs7XZ3hWHTlbXGXCqohYAMJYrM3SHGGbI4Y61bTENjdLwqHIiJ7rxFm1Xc/h8ay/PgLAAaP2VIldD7o5hhhzueGkNAG4xETlbUttJwNnFrtcEfOBs2zwmbjGRHTDMkMMVlFmaf3knE5EzWZqAC8v0aGp2raGT7Jche2KYIYdqMLagqLJ1X3x4TJC4xRB5mdgQX/QIUMJoMlt/qXAFVfomnL1cD4kEGBvPMEN3jmGGHOpEuR5mAegZoOQ5EkROJpFIXHKryXJL9uCIQGh85SJXQ56AYYYc6lhbvwzPlyESh2WCtiudBGydx8QtJrIThhlyKMvSNodLEonDekdTyTWYza4xdNJ6WB7DDNkJwww5lPXkX/bLEIliSGQgVHIpahqace5KndjloKymEReqGyCTSpDcFrSI7pRDw8y9996L2NhYqFQqREREYNasWSgvL7e5RiKRtHusX7/e5pqCggKkpqZCrVYjKioKmZmZLj/WngBdYzPOt03sHRrFlRkiMchlUoxo+2Ui2wW2mixbTEOjNAhQsV+G7MOhYWbChAn49NNPcfr0aezcuRNnz57F/fff3+66LVu24NKlS9bH7Nmzre/T6/WYNGkSIiMjkZ2djbVr12L16tVYs2aNI0snOyhoW5WJCVEjxE8hcjVE3iupV+sKiCvMaeL5MuQIPo785M8995z1z7169cLLL7+M++67D83NzZDLryfyoKAghIeHd/g5tm/fjqamJmzduhVKpRIJCQkoKirCmjVrkJGRwRNlXdjxshoAbP4lEpvlvJlckSdoC4KAQ2z+JQdwWs/M1atXsX37dowbN84myADAggULoNVqkZycjPXr18NsNlvfd/DgQaSmpkKpvH7c9ZQpU1BeXo7i4uIOv5bBYIBer7d5kPMdL2XzL5ErGNUrGBIJUFzdgMu14g2dLLnagHJdE+QyiXW1iMgeHB5mXnrpJfj5+SE0NBQlJSX48ssvbd7/2muv4bPPPsOePXswY8YMLFy4ECtXrrS+v6KiAmFhYTYfY3m7oqKiw6+5atUqaDQa6yMmJsbO3xV1xvGLNQCAoVFBotZB5O0CVXIMCAsAIO7qzIG2VZkRMUFQK2Si1UGep8thZtmyZR027d74yMnJsV7/wgsvIC8vD1lZWZDJZHjkkUdsmndfeeUVpKSkYMSIEVi4cCEyMzPx5ptv2nzNX24lWT7+ZltMixYtgk6nsz5KS0u7+m3SHbpca0C5rgkSCTCUKzNEorOcNyNm38z1EQZa0Wogz9TlnpkFCxZgxowZt7wmLi7O+metVgutVov+/ftj0KBBiImJwaFDh5CSktLhx44dOxZ6vR6VlZUICwtDeHh4uxWYqqoqAGi3YmOhVCpttqXI+Qra+mX69PCHv9KhrVlE1AnJcSHYfrgEOSJN0BYE4fr5Mr3ZL0P21eVXGUs46Q7LiorBcPM927y8PKhUKgQFBQEAUlJSsHjxYhiNRigUrXfEZGVlITIy0iY0kWs5VsrhkkSuxLIyU1imQ6PR5PRtnrOX63C51gCFjxQjY4Oc+rXJ8zmsZ+bIkSN45513kJ+fjwsXLmDfvn1IS0tDnz59rKsyX331FTZu3IjCwkKcPXsWmzZtwpIlS/DEE09YV1bS0tKgVCqRnp6OwsJCfP7551i5ciXvZHJxln6ZYTxfhsglRAerERaoRItZwLG2/z6dybLFlNQrGCo5+2XIvhwWZtRqNXbt2oV77rkHAwYMwJw5c5CQkIDvv//eGlTkcjneffddpKSkYNiwYfjv//5vZGZm4q233rJ+Ho1Gg927d+PixYtISkrCU089hYyMDGRkZDiqdLpDgiDw5F8iFyORSK6PNhBhq4lbTORIDmtmGDp0KL777rtbXjN16lRMnTq1U59r//799iqNHKxc14TqeiN8pBIMjggUuxwiapPUKxjfHL/k9AnaZrNwQ/MvwwzZH2czkd0db5uUPSA8gMvJRC7EcrbL0QvOHTp5urIW1xqa4auQ8RBNcgiGGbK7YxfZ/EvkigZFBMBXIYO+qQU/Vzlv6KS1XyYuBAofvuyQ/fFvFdmdtfmXv4ERuRQf2fU7iXKceHie5bA89suQozDMkF2ZzQIKyrgyQ+SqEp08dNJkFnD4POcxkWMxzJBdFVfXo7apBUofKfq3HZ9ORK4jyXISsJNWZk6U61Db1IIApQ+GRPKGAHIMhhmyK8st2YMjAyGX8a8XkasZGRsEqQQovdqISn2Tw7+epV9mTO8Q+PDfBHIQ/s0iu7IcxjWc/TJELilAJcfA8NYVEmdsNVnOlxnLfhlyIIYZsqsC3slE5PKS4pyz1dRsMuPI+davwfNlyJEYZshuWkxmFJZbwkyQuMUQ0U1Z5jQ5+iTg4xd1aDCaEOQrx6Bw9suQ4zDMkN38XFWHpmYz/JU+6K31E7scIrqJ5LaxBifK9Wgwtjjs6xyybDHFh0Iq5Sw9chyGGbIby/kyCVGB/IeLyIVFBqkRqVHBZBaQX1LjsK9z4OwVANxiIsdjmCG7sdzJxOZfIteX2LY6k+OgrSZDi8naYMzzZcjRGGbIbqyTshlmiFze9fNmHBNm8ktqYGgxQ+uvRN+e/g75GkQWDDNkF4YWE05V6AHwTiYid2C5o+nohWswOWDopGWEwdjeIZBIuO1MjsUwQ3bx06VaNJsEBPvKER2sFrscIrqNgeGB8Ff6oM7QgtMVtXb//JbzZcb10dr9cxP9EsMM2UXBDcMl+VsYkeuTSSXWoZO5dj5vptFosjYWs/mXnIFhhuzimLX5l1tMRO4iqZdjmoBzL1yD0WRGeKAKcaG+dv3cRB1hmCG7sNyWPZTNv0Ruw3oSsJ3HGhw813pL9rg+oVypJadgmKE7Vm9owZmqOgBcmSFyJyNigiCTSlBW04hLuka7fV5r8y+3mMhJGGbojp0o18MsAOGBKvQMVIldDhF1kp/SB4MiAgDYb3WmztBiPaYhhcMlyUkYZuiOHbc2/3JVhsjdWPtmiu3TBJxdfBUms4CYEDViQtgvQ87BMEN37BgnZRO5resTtO2zMnOwbYuJqzLkTAwzdMeO33BbNhG5F8vKzE+X9Kgz3PnQSUuY4fky5EwMM3RHdA3NuFDdAIArM0TuKFyjQnSwGmYBdzx0UtfQjBPlbf0ybP4lJ2KYoTtyvKwGANAr1BdBvgpxiyGibrHMacq+w76Zw+erYRaA3lo/hPFmAHIihhm6I5a7FoZGcVWGyF1ZJmjn3mHfjGWEAVdlyNkYZuiOHCutAQAMZ78MkduyrMzklVxDi8nc7c9jbf5lmCEnY5ihO1JQxjuZiNxd/7AABKh8UG804VQ3h05W1xmsHzuWdzKRkzHMULdV1Tbhkq4JEgmQwG0mIrclk0owKtYy2qB7fTOHz7d+3ICwAGj9lXarjagzGGao246Xtq7K9O3hDz+lj8jVENGdsGw1dfe8GW4xkZgYZqjbeL4MkedIirOcBHwNgiB0+eMPnG0dLsktJhIDwwx12/G2fpnhMdxiInJ3I2KC4COVoELfhLKarg2drNI34ezlekgkwNjeIQ6qkOjmnBJmDAYDRowYAYlEgvz8fJv3lZSUYPr06fDz84NWq8XTTz8No9Foc01BQQFSU1OhVqsRFRWFzMzMbv3mQPYjCIL1tmyuzBC5P7VChiGRgQC6fou25ZbswRGBPG+KROGUMPPiiy8iMjKy3fMmkwnTpk1DfX09fvzxR+zYsQM7d+7EwoULrdfo9XpMmjQJkZGRyM7Oxtq1a7F69WqsWbPGGaXTTVy81oir9Ub4SCUYGB4gdjlEZAeJva5vNXUF5zGR2BweZr799ltkZWVh9erV7d6XlZWFkydP4sMPP8TIkSMxceJEvPXWW9i4cSP0ej0AYPv27WhqasLWrVuRkJCAP/zhD1i8eDHWrFnD1RkRWVZlBkYEQCWXiVwNEdlDclz3TgK2rMyM68swQ+JwaJiprKzE3Llz8cEHH8DXt/0o+IMHDyIhIcFm1WbKlCkwGAzIzc21XpOamgqlUmlzTXl5OYqLizv8ugaDAXq93uZB9mUZY8AtJiLPkdgWZk5X1kLf1NypjymracSF6gbIpBIkx7FfhsThsDAjCALS09Mxb948JCUldXhNRUUFwsLCbJ4LDg6GQqFARUXFTa+xvG255pdWrVoFjUZjfcTExNzpt0O/YLktezgPyyPyGD0DVIgN8YUgAHmdHDpp2WIaGqVBgEruwOqIbq7LYWbZsmWQSCS3fOTk5GDt2rXQ6/VYtGjRLT+fRCJp95wgCDbP//Iay/ZSRx8LAIsWLYJOp7M+SktLu/pt0i2YzQIKyywzmYLELYaI7CqpbXUmt5NbTTxfhlxBl086W7BgAWbMmHHLa+Li4vD666/j0KFDNttDAJCUlISHH34Y27ZtQ3h4OA4fPmzz/mvXrqG5udm6+hIeHt5uBaaqqgoA2q3YWCiVynZfl+zn3JV61BpaoJJL0T/MX+xyiMiOknqFYNfRMmR3oglYEAQcbDtfhs2/JKYuhxmtVgutVnvb695++228/vrr1rfLy8sxZcoUfPLJJxgzZgwAICUlBStWrMClS5cQEREBoLUpWKlUIjEx0XrN4sWLYTQaoVAorNdERkYiLi6uq+WTHRS09csMidTAR8ajiog8iWVlJr+0Bs0mM+S3+G+85GoDynVNkMsk1o8jEoPDXoliY2ORkJBgffTv3x8A0KdPH0RHRwMAJk+ejMGDB2PWrFnIy8vD3r178fzzz2Pu3LkIDGw97yAtLQ1KpRLp6ekoLCzE559/jpUrVyIjI+Om20zkWMdKOVySyFP17eGPQJUPGptN+OnSrW+esGwxjYgJgq+CI01IPKL+Wi2TyfDNN99ApVJh/PjxePDBB3HffffZ3Mat0Wiwe/duXLx4EUlJSXjqqaeQkZGBjIwMESv3btfHGDDMEHkaqVRiHW1wu62mAzxfhlyE06J0XFxch+fCxMbG4uuvv77lxw4dOhT79+93VGnUBc0mM06Ut/62xtuyiTxTYq9gfHeqCrkXruKxu+I7vEYQBOv5Mil9bt96QORIbHigLvm5sg6GFjMClD6ID/UTuxwicgDrBO1bDJ08e7kel2sNUPhIMTI2yInVEbXHMENdYtliGhqtgVTKniUiTzQ8JghymQRVtQZcvNbx0EnLXUyJscE8BZxExzBDXXKsbYzBUPbLEHkslVyGhKjW/8ZvNtrAOsKA58uQC2CYoS6xrMwMZ78MkUezbjV1MEHbbBZ4WB65FIYZ6rSmZhNOV9QC4J1MRJ7OMkE7t4M7mk5X1uJaQzPUchlvBCCXwDBDnfbTJT1azAJC/RSIClKLXQ4ROVDSDUMndQ22QyctqzLJ8SFQ+PBlhMTHv4XUacdv6JfhgYVEnk3rr0S8tvWOxaMltqszPF+GXA3DDHXaMetheUGi1kFEzpFo7Zu53gRsMgs4fJ7Nv+RaGGao0ywrM8PZL0PkFZLjrp83Y3GyXI/aphYEKH0wJDJQrNKIbDDMUKfUGVpw9nIdAN6WTeQtLE3A+aU1MLaYAQAH2s6XGR0fwkGz5DL4N5E6pbBMB0EAIjQq9AxQiV0OETlBnx5+CPaVw9Bixony1pXZ6yMMuMVEroNhhjqFwyWJvI9EIrH2zeReuIZmkxnZ51v7ZxhmyJUwzFCnWE7+ZfMvkXe5PkH7Ko5f1KHeaEKQrxyDwtkvQ66DYYY6pcDa/BskbiFE5FRJN6zMHGrbYhobH8rZbORSGGbotq7VG1FytQEAMDSK20xE3iQhSgOFTIordUZ8mlMKgFtM5HoYZui2jpe1rsrEhfpC4ysXuRoiciaVXGbtlbtQ3fpLDcMMuRqGGbqt46U1ANgvQ+StEtvOmwEArb8C/Xr6i1gNUXsMM3RblpUZ3slE5J2S2s6bAYCxvUM5zoRcDsMM3dZxjjEg8mqW27MBbjGRa2KYoVuq1DehUm+AVAIkRPFWTCJvFOKnQErvUAQofTBhQE+xyyFqx0fsAsi1HWvrl+nXMwC+Cv51IfJW76cno6nZhGA/hdilELXDVye6pQL2yxARALVCBrVCJnYZRB3iNhPd0vWTfxlmiIjINTHM0E0JgsDmXyIicnkMM3RTpVcbUdPQDLlMgoERAWKXQ0RE1CGGGbqp42U1AIBBEYFQ+nCvnIiIXBPDDN3U8bZ+Gc5jIiIiV8YwQzdluS2bk7KJiMiVMcxQh0xmAYWW27JjuDJDRESui2GGOnT+Sh3qjSao5TL07cGhckRE5LoYZqhDx0pbV2WGRAbCR8a/JkRE5Lr4KkUd4vkyRETkLhhmqEOWk3+Hs1+GiIhcnFPCjMFgwIgRIyCRSJCfn2/zPolE0u6xfv16m2sKCgqQmpoKtVqNqKgoZGZmQhAEZ5TulZpNZpy8pAfAlRkiInJ9Thk0+eKLLyIyMhLHjh3r8P1btmzB1KlTrW9rNNdXA/R6PSZNmoQJEyYgOzsbRUVFSE9Ph5+fHxYuXOjw2r3R6YpaGFvMCFD5oFeIr9jlEBER3ZLDw8y3336LrKws7Ny5E99++22H1wQFBSE8PLzD923fvh1NTU3YunUrlEolEhISUFRUhDVr1iAjIwMSiaTdxxgMBhgMBuvber3ePt+Mlzh+w3BJqbT9z5eIiMiVOHSbqbKyEnPnzsUHH3wAX9+b/4a/YMECaLVaJCcnY/369TCbzdb3HTx4EKmpqVAqldbnpkyZgvLychQXF3f4+VatWgWNRmN9xMTE2O178gZs/iUiInfisDAjCALS09Mxb948JCUl3fS61157DZ999hn27NmDGTNmYOHChVi5cqX1/RUVFQgLC7P5GMvbFRUVHX7ORYsWQafTWR+lpaV2+I68h2VlZng0m3+JiMj1dXmbadmyZVi+fPktr8nOzsaBAweg1+uxaNGiW177yiuvWP88YsQIAEBmZqbN87/cSrI0/3a0xQQASqXSZiWHOq+p2YTTlbUAgKFcmSEiIjfQ5TCzYMECzJgx45bXxMXF4fXXX8ehQ4fahYqkpCQ8/PDD2LZtW4cfO3bsWOj1elRWViIsLAzh4eHtVmCqqqoAoN2KDd25E+V6mMwCtP4KRGpUYpdDRER0W10OM1qtFlqt9rbXvf3223j99detb5eXl2PKlCn45JNPMGbMmJt+XF5eHlQqFYKCggAAKSkpWLx4MYxGIxQKBQAgKysLkZGRiIuL62r5dBs39svcbOWLiIjIlTjsbqbY2Fibt/39W+f79OnTB9HR0QCAr776ChUVFUhJSYFarca+ffuwZMkSPPHEE9YVnbS0NCxfvhzp6elYvHgxfv75Z6xcuRJLly7li60DFNxwJxMREZE7cMo5Mzcjl8vx7rvvIiMjA2azGb1790ZmZibmz59vvUaj0WD37t2YP38+kpKSEBwcjIyMDGRkZIhYuec6Zl2ZYZghIiL3IBG84ChdvV4PjUYDnU6HwMBAsctxWbVNzRi2PAuCAOS8MhFafzZRExGReDr7+s3ZTGRVUKaDIABRQWoGGSIichsMM2TFfhkiInJHDDNkZTksbyjDDBERuRGGGbKyNP8O52F5RETkRhhmCABQXWfAxWuNAICEKK7MEBGR+2CYIQCtzb8A0FvrB41aLnI1REREnccwQwDYL0NERO6LYYYA2I4xICIicicMMwRBEHCsbWVmOFdmiIjIzTDMECr1BlyuNUAqAYZEMswQEZF7YZgh6y3Z/cMCoFbIxC2GiIioixhm6IZ+Ga7KEBGR+2GYIeudTGz+JSIid8Qw4+UEQbCGGZ78S0RE7ohhxsuVXG2ArrEZCpkUA8IDxC6HiIioyxhmvJzlluxBEQFQ+PCvAxERuR++enm546U1ANgvQ0RE7othxssdL7M0//JOJiIick8MM17MZBZQWMY7mYiIyL0xzHixs5fr0GA0wVchQ9+e/mKXQ0RE1C0MM17sWFu/TEKkBjKpRNxiiIiIuolhxosVsF+GiIg8AMOMF7Pclj2UYYaIiNwYw4yXMraY8VO5HgBP/iUiIvfGMOOlTlfUwmgyQ6OWo1eor9jlEBERdRvDjJc6XlYDoLVfRiJh8y8REbkvhhkvdby0rV8miv0yRETk3hhmvNSxizUAeFgeERG5P4YZL9RoNOHnqjoAwPAYrswQEZF7Y5jxQicv6WAyC+gRoER4oErscoiIiO4Iw4wXOtbWLzMsis2/RETk/hhmvNBx9ssQEZEHcWiYiYuLg0QisXm8/PLLNteUlJRg+vTp8PPzg1arxdNPPw2j0WhzTUFBAVJTU6FWqxEVFYXMzEwIguDI0j3a8baTf4exX4aIiDyAj6O/QGZmJubOnWt929//+nRmk8mEadOmoUePHvjxxx9RXV2N2bNnQxAErF27FgCg1+sxadIkTJgwAdnZ2SgqKkJ6ejr8/PywcOFCR5fvcfRNzTh3pR5A6zYTERGRu3N4mAkICEB4eHiH78vKysLJkydRWlqKyMhIAMBbb72F9PR0rFixAoGBgdi+fTuampqwdetWKJVKJCQkoKioCGvWrEFGRkaHPR8GgwEGg8H6tl6vd8j3tvenSvx45opDPrejXKlrXfWKClIj1F8pcjVERER3zuFh5o033sBrr72GmJgYPPDAA3jhhRegUCgAAAcPHkRCQoI1yADAlClTYDAYkJubiwkTJuDgwYNITU2FUqm0uWbRokUoLi5GfHx8u6+5atUqLF++3NHfGnIvXMOWfxU7/Os4QmKvYLFLICIisguHhplnnnkGo0aNQnBwMI4cOYJFixbh/Pnz2LRpEwCgoqICYWFhNh8THBwMhUKBiooK6zVxcXE211g+pqKiosMws2jRImRkZFjf1uv1iImJsee3BgAY2zsU7ngzkEImwwNJ0WKXQUREZBddDjPLli277apHdnY2kpKS8Nxzz1mfGzZsGIKDg3H//ffjjTfeQGhoKAB0uE0kCILN87+8xtL8e7PbipVKpc1KjqP8un8P/Lp/D4d/HSIiIrq5LoeZBQsWYMaMGbe85pcrKRZjx44FAJw5cwahoaEIDw/H4cOHba65du0ampubrasv4eHh1lUai6qqKgBot6pDRERE3qfLYUar1UKr1Xbri+Xl5QEAIiIiAAApKSlYsWIFLl26ZH0uKysLSqUSiYmJ1msWL14Mo9Fo7bXJyspCZGTkTUMTEREReQ+HnTNz8OBB/PWvf0V+fj7Onz+PTz/9FE8++STuvfdexMbGAgAmT56MwYMHY9asWcjLy8PevXvx/PPPY+7cuQgMDAQApKWlQalUIj09HYWFhfj888+xcuXKm97JRERERN7FYQ3ASqUSn3zyCZYvXw6DwYBevXph7ty5ePHFF63XyGQyfPPNN3jqqacwfvx4qNVqpKWlYfXq1dZrNBoNdu/ejfnz5yMpKQnBwcHIyMiwafAlIiIi7yURvOAoXb1eD41GA51OZ13xISIiItfW2ddvzmYiIiIit8YwQ0RERG6NYYaIiIjcGsMMERERuTWGGSIiInJrDDNERETk1hhmiIiIyK0xzBAREZFbc9gJwK7Eci6gXq8XuRIiIiLqLMvr9u3O9/WKMFNbWwsAiImJEbkSIiIi6qra2lpoNJqbvt8rxhmYzWaUl5cjICDA7sMp9Xo9YmJiUFpaylEJDsSfs3Pw5+wc/Dk7B3/OzuHIn7MgCKitrUVkZCSk0pt3xnjFyoxUKkV0dLRDv0ZgYCD/Y3EC/pydgz9n5+DP2Tn4c3YOR/2cb7UiY8EGYCIiInJrDDNERETk1hhm7pBSqcSrr74KpVIpdikejT9n5+DP2Tn4c3YO/pydwxV+zl7RAExERESeiyszRERE5NYYZoiIiMitMcwQERGRW2OYISIiIrfGMENERERujWHmDrz77ruIj4+HSqVCYmIifvjhB7FL8iirVq1CcnIyAgIC0LNnT9x33304ffq02GV5vFWrVkEikeDZZ58VuxSPVFZWhpkzZyI0NBS+vr4YMWIEcnNzxS7Lo7S0tOCVV15BfHw81Go1evfujczMTJjNZrFLc2v79+/H9OnTERkZCYlEgi+++MLm/YIgYNmyZYiMjIRarcbdd9+NEydOOKU2hplu+uSTT/Dss89iyZIlyMvLw69+9Sv89re/RUlJidileYzvv/8e8+fPx6FDh7B79260tLRg8uTJqK+vF7s0j5WdnY0NGzZg2LBhYpfika5du4bx48dDLpfj22+/xcmTJ/HWW28hKChI7NI8yhtvvIH169fjnXfewU8//YS//OUvePPNN7F27VqxS3Nr9fX1GD58ON55550O3/+Xv/wFa9aswTvvvIPs7GyEh4dj0qRJ1mHPDiVQt4wePVqYN2+ezXMDBw4UXn75ZZEq8nxVVVUCAOH7778XuxSPVFtbK/Tr10/YvXu3kJqaKjzzzDNil+RxXnrpJeGuu+4SuwyPN23aNGHOnDk2z/3hD38QZs6cKVJFngeA8Pnnn1vfNpvNQnh4uPDnP//Z+lxTU5Og0WiE9evXO7wersx0g9FoRG5uLiZPnmzz/OTJk3HgwAGRqvJ8Op0OABASEiJyJZ5p/vz5mDZtGiZOnCh2KR7rH//4B5KSkvDAAw+gZ8+eGDlyJDZu3Ch2WR7nrrvuwt69e1FUVAQAOHbsGH788Uf87ne/E7kyz3X+/HlUVFTYvC4qlUqkpqY65XXRK6Zm29uVK1dgMpkQFhZm83xYWBgqKipEqsqzCYKAjIwM3HXXXUhISBC7HI+zY8cOHD16FNnZ2WKX4tHOnTuHdevWISMjA4sXL8aRI0fw9NNPQ6lU4pFHHhG7PI/x0ksvQafTYeDAgZDJZDCZTFixYgUeeughsUvzWJbXvo5eFy9cuODwr88wcwckEonN24IgtHuO7GPBggU4fvw4fvzxR7FL8TilpaV45plnkJWVBZVKJXY5Hs1sNiMpKQkrV64EAIwcORInTpzAunXrGGbs6JNPPsGHH36Ijz76CEOGDEF+fj6effZZREZGYvbs2WKX59HEel1kmOkGrVYLmUzWbhWmqqqqXSqlO/enP/0J//jHP7B//35ER0eLXY7Hyc3NRVVVFRITE63PmUwm7N+/H++88w4MBgNkMpmIFXqOiIgIDB482Oa5QYMGYefOnSJV5JleeOEFvPzyy5gxYwYAYOjQobhw4QJWrVrFMOMg4eHhAFpXaCIiIqzPO+t1kT0z3aBQKJCYmIjdu3fbPL97926MGzdOpKo8jyAIWLBgAXbt2oXvvvsO8fHxYpfkke655x4UFBQgPz/f+khKSsLDDz+M/Px8Bhk7Gj9+fLvjBYqKitCrVy+RKvJMDQ0NkEptX95kMhlvzXag+Ph4hIeH27wuGo1GfP/99055XeTKTDdlZGRg1qxZSEpKQkpKCjZs2ICSkhLMmzdP7NI8xvz58/HRRx/hyy+/REBAgHUlTKPRQK1Wi1yd5wgICGjXh+Tn54fQ0FD2J9nZc889h3HjxmHlypV48MEHceTIEWzYsAEbNmwQuzSPMn36dKxYsQKxsbEYMmQI8vLysGbNGsyZM0fs0txaXV0dzpw5Y337/PnzyM/PR0hICGJjY/Hss89i5cqV6NevH/r164eVK1fC19cXaWlpji/O4fdLebC//e1vQq9evQSFQiGMGjWKtwzbGYAOH1u2bBG7NI/HW7Md56uvvhISEhIEpVIpDBw4UNiwYYPYJXkcvV4vPPPMM0JsbKygUqmE3r17C0uWLBEMBoPYpbm1ffv2dfhv8uzZswVBaL09+9VXXxXCw8MFpVIp/PrXvxYKCgqcUptEEATB8ZGJiIiIyDHYM0NERERujWGGiIiI3BrDDBEREbk1hhkiIiJyawwzRERE5NYYZoiIiMitMcwQERGRW2OYISIiIrfGMENERERujWGGiIiI3BrDDBEREbm1/x+FCqA/fQfzBQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 使用Matplotlib绘制奖励值的曲线图\n",
    "plt.plot(return_list)\n",
    "plt.title(\"reward\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f2526ca7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAMsAAADSCAYAAADkIjRgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8+yak3AAAACXBIWXMAAAsTAAALEwEAmpwYAAARYElEQVR4nO3deYyc9X3H8fd3ZnbWXhuv8YFxfGBjTIhTlaMWh0AVJaGlJg1IcRAIBZpaIW2CREoSYlIpCVFTkaoKSVQaSgoJjShHCSWIo9y0VC0GcwawgDUY8Nqw+MKsvZ7z2z+e35rxZo/feOfYmf28pJGf+T0z8/xmx5/5Pdc8X3N3RGRsqWZ3QKRVKCwikRQWkUgKi0gkhUUkksIiEklhaTIz6zezI2v1WDM73cw216Z3YGZuZkfV6vVaWabZHZhMzGwTMA8oVTQf7e5bYp7v7tPr0S+Jo7A03p+5+8PN7oRUT6thTVa5mmNmvzSza83sXjP70MzWmdmyER67ysxeCY/rNbNvDHndr5tZn5ltNbMvVrR3mtk/mNnbZvaemV1nZlMr5n8zPGeLmf1F/f8CrUNhmXjOB64CDgV6gB+M8LgbgC+7+yHA7wGPVsw7HOgGFgBrgGvN7NAw72rgaOA44KjwmO8AmNlZwDeAM4HlwKdr9abagcLSeHeZ2a5wu2uY+f/h7k+5exG4meQ/9XAKwAozm+HuO9392SHzvu/uBXe/D+gHPm5mBlwC/LW773D3D4G/IwkowHnAL9z9JXffA3xvnO+1rSgsjXeuu88Mt3OHmf9uxfReYKSN+s8Bq4C3zOy/zOyUinnbQ9iGvs5coAt4ZjCwwH+GdoCPAe9UPO+tyPc0KWgDv0W5+9PAOWbWAVwK3A4sGuNp24AB4JPu3jvM/K1DXmNxLfraLjSytCAzy5rZhWbW7e4FYDdQHut57l4Gfg5cY2aHhddaYGZ/Eh5yO/DnZrbCzLqA79bpLbQkhaV1fQHYZGa7gb8ELox83rdIdhw8GZ77MPBxAHe/H/gxyc6CHg7caTDpmX78JRJHI4tIpLqExczOMrNXzazHzNbWYxkijVbz1TAzSwOvkRzY2gw8DVzg7q/UdEEiDVaPkeVEoMfd33D3PHArcE4dliPSUPUIywIOPLC1ObSJtLSmHZQ0s0tITr1g2rRpf3DMMcc0qysi+23atIlt27bZcPPqEZZeDjwKvDC0HcDdrweuB1i5cqWvX7++Dl0Rqc7KlStHnFeP1bCngeVmttTMsiQn6d1dh+WINFTNRxZ3L5rZpcADQBq40d1frvVyRBqtLtss4bTw++rx2iLNoiP4IpEUFpFICotIJIVFJJLCIhJJYRGJpLCIRFJYRCIpLCKRFBaRSAqLSCSFRSSSwiISSWERiaSwiERSWEQiKSwikRQWkUgKi0gkhUUkksIiEklhEYmksIhEUlhEIo0ZFjO70cz6zOylirZZZvaQmb0e/j00tJuZ/TQUMXrRzE6oZ+dFGilmZPklcNaQtrXAI+6+HHgk3Af4U2B5uF0C/Kw23RRpvjHD4u7/DewY0nwOcFOYvgk4t6L9Xz3xJDDTzObXqK8iTXWw2yzz3H1rmH4XmBemowsZmdklZrbezNa///77B9kNkcYZ9wa+J0Upqy5M6e7Xu/tKd185d+7c8XZDpO4ONizvDa5ehX/7QntUISORVnSwYbkbuDhMXwz8pqL9orBX7GTgg4rVNZGWNmZ9FjO7BTgdmGNmm4HvAlcDt5vZGuAt4Lzw8PuAVUAPsBf4Yh36LNIUY4bF3S8YYdanhnmsA18db6dEJqKmVStuVdsKBb6/dSvT02kOy2Q4rKODWek0h6bTzMxkmJ3JMDudxmzYgrfSwhSWKvUVi/xi+3b6y+X9bWkga0Y2lWJlVxf3L19OR/O6KHWisFRpR6lE0Q/cU14CBtwZKJXYWSo1p2NSdzqRskpv5HLs85EPKx2ZzeqP2qb0uVbB3SmPEhSAxQpL25oQq2G7d+/mwQcfbHY3orwwZQpMmzbi/L5Nm3how4YG9khqaffu3SPOmxBhmTZtGieddFKzuxHlf3fsgB1Dzyv9yOKFCzlp1qwG9khqadooX4QTIizpdJru7u5md2NM7s7WXbtGnJ8CPjFjRku8FxleOp0ecZ5Wr6vUWyiMOC8FLOjQTuN2pbDUkJnRPco3k7Q2haUKJSBXcTByOCkduW9bCksV8u58OMpBx0NSKaan9CdtV/pkq7C7VGJTPj/i/Nnh3DBpTwpLFcru5Ec5KDk1lWKqRpa2pU+2CmVG//20oT9oO9NnW4X+cpnSKCPL4R0dOjW/jSksVegrFCiMEpal2Szacdy+FJYqvJ3Pj3rG8aGZDBpX2pfCUoWxrveU0SpYW1NYIrn7qKtgkBxn0TZL+1JYqrCzWBxxXhpYMWVK4zojDaewVOHtUU6iNGCWDki2NYWlCm/lcqPOz2oVrK0pLFUYbYslbaaTKNtcTDGjRWb2mJm9YmYvm9lloX1SFTQqAblRNvDnZTLM129Z2lrMyFIEvu7uK4CTga+a2QomWUGjwhhnHHel0zrjuM3FFDPa6u7PhukPgQ0kNVcmVUGj/lKJt0c54ziDjrO0u6q+Cs1sCXA8sI5xFjRqtWJGJWBglB9+ZVMpHb1vc9FhMbPpwK+Br7n7AdeLOZiCRq1WzKjsPuob/MSUKRpZ2lxUWMysgyQoN7v7naF5UhU02lMuM9qFWedkMtq12OZi9oYZcAOwwd1/VDFrUhU02l4skh9lNaxLG/dtL+aQ86nAF4Dfmtnzoe3bTLKCRpvyeQZG2XVsoPPC2lxMMaP/gRG3XSdNQaOxfiF5jM4La3tad4gw1hnHurje5KAz/yLtLBZJUeJoXuNE1pGlwAscy/McR4ksz+zdy7R0mjmh8tfUVIqMGSm0etYuFJZIPfv2cB63cwnXcwgfApCjk7s4l3/kUtb29pIxY0Y6TXc6zfxMhsWdnRyRzbI0m+WIzk7mZzLMyWTozmToQOeTtRqFJdLi4v9xHv9EFwP726aQYzV30MsCbuEC8g7bikW2FYtszOVgzx4gbPyT7DHrTqeZncmwqKODRdksSzo7OTKbZWE2y9xw3bGpqRQdGpUmHIUl0tncy0BFUAZlKHE293IHqymQHfa5g0ds+8tl+stlegsFXhz46LUM6DDjkFSKGek0HwtBOiKbZWlnJ0uyWQ7v6GBuJsOMdJqsGR1mClKDKSyROskNE5WP5ll1JzAcwEkuDbu9VGJ7qcSb+fz+UQmSHQhd4dKwczIZFmazLAqrd0s7O1mUzXBEpsRhmRQd6S6SY8hSawpLpK6u49i1687faXegh2WU6ngRpDIfjUrvFou8tG/f/nkz2cVFdhurUk+wPV1kxvQTmTfvm0ydeqxGnhpTWCKYGXPmfIndux+iv/+J/e0O9LKAG1lT17CMZBr9fJsfcLo/TqrkFEuwY0cP/f1PsmzZHXR1Hd/wPrUzHWeJlMnMY8mSXzFnzpeg4wj2pA7nCf6ItVzN6yxn5OO29fPHPMjpPE5qyCpgPv8GW7Z8j3J55J8USPU0skQyM7LZxSxe/DPmFneyNZ9nbyHLpfkyb+ZyvJnPs6VQYGuhwAelEgPl8qi/rKyFT/Ly7wRl0MDA87gPwAg7HaR6CksVkm2ANF0dc1jWActIju5Dskq2z52BcpkdxSJbCgV683nezOd5I5djcz5Pb6FAX7HI3nKZgXKZ0csija1IBmf4MS3ZyNc2Sy0pLOM0uBFtQJcZXakUszMZllecKzZ4usw+d/pLJd4tFnm3UOCtXI6N+Txv5XL0VoxKeyNHpYf5NKu474BjP6FXzJz5OVKp6TV8p6KwNICZkTUjC8lxlOxHq0buTplk1/FAucz2EKTeQoE3czneyOfZnM/zTj7PtmKRPeUye8Oo9BwncB1f5hJ+zjT2hHEkw8yZ53L44Vdgpk3SWlJYmszMSANTzZiaSjFrhFEpFy6Y0RfCtCmfZ2MuR2/+K9yWP5nTeILTuozp00+lu3sV6fSM5r2pNqWwTHCVo9IhI45Ky4DPq+pYnSksLaxyVJL601eRSCSFRSSSwiISSWERiaSwiERSWEQiKSwikWKuSDnFzJ4ysxdCfZarQvtSM1sX6rDcZmbZ0N4Z7veE+Uvq/B5EGiJmZMkBZ7j7scBxwFnhsqw/BK5x96OAncCa8Pg1wM7Qfk14nEjLi6nP4u7eH+52hJsDZwB3hPah9VkG67bcAXzK9PtWaQOxV9FPh+sc9wEPARuBXe4+WOu6sgbL/vosYf4HwOwa9lmkKaLC4u4ldz+OpHzEicAx411wqxUzEqlqb5i77wIeA04hKX83eCJmZQ2W/fVZwvxuYPswr9VSxYxEYvaGzTWzmWF6KnAmSV3Jx4DV4WFD67MM1m1ZDTzqXucfo4s0QMwp+vOBm8wsTRKu2939HjN7BbjVzP4WeI6k4BHh31+ZWQ+wAzi/Dv0WabiY+iwvkhRdHdr+Bsn2y9D2fcDna9I7kQlER/BFIiksIpEUFpFICotIJIVFJJLCIhJJYRGJpLCIRFJYRCIpLCKRFBaRSAqLSCSFRSSSwiISSWERiaSwiERSWEQiKSwikRQWkUgKi0gkhUUkksIiEklhEYmksIhEig5LuJL+c2Z2T7ivYkYyqVQzslxGco3jQSpmJJNKbH2WhcDZwL+E+4aKGckkEzuy/Bi4AiiH+7MZZzEj1WeRVhNTcuIzQJ+7P1PLBas+i7SamJITpwKfNbNVwBRgBvATQjGjMHoMV8xo82jFjERaTUwB1ivdfaG7LyGptfKou1+IihnJJDOe4yzfAi4PRYtmc2Axo9mh/XJg7fi6KDIxxKyG7efujwOPh2kVM5JJRUfwRSIpLCKRFBaRSAqLSCSFRSSSwiISSWERiaSwiERSWEQiKSwikRQWkUgKi0gkhUUkksIiEklhEYmksIhEUlhEIiksIpEUFpFICotIJIVFJJLCIhJJYRGJpLCIRFJYRCIpLCKRFBaRSDYRLnBvZh8Crza5G3OAbZN4+epD4gh3H7ZgUFUXBq+jV919ZTM7YGbrm9mHZi9ffRibVsNEIiksIpEmSliub3YHaH4fmr18UB9GNSE28EVawUQZWUQmvKaHxczOMrNXzazHzOpWf9LMbjSzPjN7qaJtlpk9ZGavh38PDe1mZj8NfXrRzE6owfIXmdljZvaKmb1sZpc1sg9mNsXMnjKzF8LyrwrtS81sXVjObWaWDe2d4X5PmL9knH+Cyr6kzew5M7unWX04KO7etBuQBjYCRwJZ4AVgRZ2W9YfACcBLFW1/D6wN02uBH4bpVcD9gAEnA+tqsPz5wAlh+hDgNWBFo/oQXmd6mO4A1oXXvR04P7RfB/xVmP4KcF2YPh+4rYafxeXAvwH3hPsN78NB9bupC4dTgAcq7l8JXFnH5S0ZEpZXgflhej7J8R6AfwYuGO5xNezLb4Azm9EHoAt4FjiJ5ABgZujnATwAnBKmM+FxVoNlLwQeAc4A7gkhbmgfDvbW7NWwBcA7Ffc3h7ZGmefuW8P0u8C8RvQrrE4cT/Lt3rA+hNWf54E+4CGSUX2XuxeHWcb+5Yf5H5CUcB+vHwNXAOVwf3YT+nBQmh2WCcOTr6+67xo0s+nAr4GvufvuRvbB3UvufhzJt/uJwDH1WtZwzOwzQJ+7P9PI5dZKs8PSCyyquL8wtDXKe2Y2HyD821fPfplZB0lQbnb3O5vRBwB33wU8RrLKM9PMBk97qlzG/uWH+d3A9nEu+lTgs2a2CbiVZFXsJw3uw0FrdlieBpaHvSFZko24uxu4/LuBi8P0xSTbEYPtF4U9UicDH1SsKh0UMzPgBmCDu/+o0X0ws7lmNjNMTyXZXtpAEprVIyx/sF+rgUfDyHfQ3P1Kd1/o7ktIPutH3f3CRvZhXJq1sVSxwbeKZM/QRuBv6ricW4CtQIFkvXgNyfrvI8DrwMPArPBYA64NffotsLIGyz+NZBXrReD5cFvVqD4Avw88F5b/EvCd0H4k8BTQA/w70Bnap4T7PWH+kTX+PE7no71hTelDtTcdwReJ1OzVMJGWobCIRFJYRCIpLCKRFBaRSAqLSCSFRSSSwiIS6f8BGc8MOi+3kqgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 216x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 重置环境，开始新的一轮游戏\n",
    "observation, _ = env.reset()\n",
    "# 创建GymHelper对象来辅助显示\n",
    "gym_helper = GymHelper(env, figsize = (3, 3))\n",
    "episode_rewards = []\n",
    "\n",
    "# 开始游戏\n",
    "for i in range(1000):\n",
    "    # 渲染环境，title为当前步骤数\n",
    "    gym_helper.render(title = str(i))\n",
    "    \n",
    "    # 找到当前状态下的最优动作\n",
    "    action = mbpo.agent.choose_action(observation)\n",
    "    # 执行action，获取新的信息\n",
    "    observation, reward, terminated, truncated, info = env.step(action)\n",
    "    episode_rewards.append(reward)\n",
    "    \n",
    "    # 如果游戏结束，则结束当前循环\n",
    "    if terminated or truncated:\n",
    "        break\n",
    "\n",
    "# 游戏结束\n",
    "gym_helper.render(title = \"Finished\")\n",
    "# 关闭环境\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "629dbc26",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
